Skip to content

Commit c14f017

Browse files
disqdisq
authored
feat: Transform objects (#34)
Fixes #5
1 parent c7b301d commit c14f017

5 files changed

Lines changed: 249 additions & 7 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,4 +43,4 @@ npm run format
4343

4444
# Lint
4545
npm run lint
46-
```
46+
```

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
"format": "prettier --write 'src/**/*.ts'",
1818
"format:check": "prettier --check 'src/**/*.ts'",
1919
"lint": "eslint --max-warnings 0 --ext .ts src",
20+
"lint:fix": "eslint --max-warnings 0 --ext .ts --fix src",
2021
"test": "ava"
2122
},
2223
"description": "This is the high-level package to use for developing CloudQuery plugins in JavaScript",

‎src/transformers/openapi.test.ts‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,32 +52,26 @@ test('should parse spec as expected', (t) => {
5252
createColumn({
5353
name: 'string',
5454
type: new Utf8(),
55-
description: '',
5655
}),
5756
createColumn({
5857
name: 'number',
5958
type: new Int64(),
60-
description: '',
6159
}),
6260
createColumn({
6361
name: 'integer',
6462
type: new Int64(),
65-
description: '',
6663
}),
6764
createColumn({
6865
name: 'boolean',
6966
type: new Bool(),
70-
description: '',
7167
}),
7268
createColumn({
7369
name: 'object',
7470
type: new JSONType(),
75-
description: '',
7671
}),
7772
createColumn({
7873
name: 'array',
7974
type: new JSONType(),
80-
description: '',
8175
}),
8276
];
8377

‎src/transformers/transform.test.ts‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
import { Utf8, Int64, Bool, List, Field, Float64, DataType } from '@apache-arrow/esnext-esm';
2+
import test from 'ava';
3+
4+
import { Column, createColumn } from '../schema/column.js';
5+
import { JSONType } from '../types/json.js';
6+
7+
import { objectToColumns } from './transform.js';
8+
9+
test('should parse object as expected', (t) => {
10+
const expectedColumns: Column[] = [
11+
createColumn({
12+
name: 'string',
13+
type: new Utf8(),
14+
}),
15+
createColumn({
16+
name: 'number',
17+
type: new Int64(),
18+
}),
19+
createColumn({
20+
name: 'float',
21+
type: new Float64(),
22+
}),
23+
createColumn({
24+
name: 'boolean',
25+
type: new Bool(),
26+
}),
27+
createColumn({
28+
name: 'object',
29+
type: new JSONType(),
30+
}),
31+
createColumn({
32+
name: 'array',
33+
type: new List(new Field('element', new Utf8())),
34+
}),
35+
];
36+
37+
const columns = objectToColumns({
38+
string: 'test',
39+
number: 1,
40+
float: 3.14,
41+
boolean: false,
42+
object: { inner: 'foo' },
43+
array: ['foo', 'bar'],
44+
});
45+
t.deepEqual(columns, expectedColumns);
46+
});
47+
48+
test('should parse object with custom types', (t) => {
49+
const expectedColumns: Column[] = [
50+
createColumn({
51+
name: 'string',
52+
type: new Utf8(),
53+
}),
54+
createColumn({
55+
name: 'float',
56+
type: new Float64(),
57+
}),
58+
];
59+
60+
const columns = objectToColumns(
61+
{
62+
string: 'test',
63+
float: 1,
64+
},
65+
{
66+
getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined {
67+
if (key === 'float') return new Float64();
68+
return undefined;
69+
},
70+
},
71+
);
72+
t.deepEqual(columns, expectedColumns);
73+
});
74+
75+
test('should parse object with custom types and allow skip columns in type transformer', (t) => {
76+
const expectedColumns: Column[] = [
77+
createColumn({
78+
name: 'string',
79+
type: new Utf8(),
80+
}),
81+
];
82+
83+
const columns = objectToColumns(
84+
{
85+
string: 'test',
86+
skip: 'test',
87+
},
88+
{
89+
getTypeFromValue: function (key: string, value: unknown): DataType | null | undefined {
90+
return key === 'skip' ? null : undefined;
91+
},
92+
},
93+
);
94+
t.deepEqual(columns, expectedColumns);
95+
});
96+
97+
test('should parse object and skip columns', (t) => {
98+
const expectedColumns: Column[] = [
99+
createColumn({
100+
name: 'string',
101+
type: new Utf8(),
102+
}),
103+
];
104+
105+
const columns = objectToColumns(
106+
{
107+
string: 'test',
108+
skip: 'test',
109+
},
110+
{
111+
skipColumns: ['skip'],
112+
},
113+
);
114+
t.deepEqual(columns, expectedColumns);
115+
});
116+
117+
test('should parse object and set PKs', (t) => {
118+
const expectedColumns: Column[] = [
119+
createColumn({
120+
name: 'id',
121+
type: new Utf8(),
122+
primaryKey: true,
123+
}),
124+
createColumn({
125+
name: 'string',
126+
type: new Utf8(),
127+
}),
128+
];
129+
130+
const columns = objectToColumns(
131+
{
132+
id: 'the-id',
133+
string: 'test',
134+
},
135+
{
136+
primaryKeys: ['id'],
137+
},
138+
);
139+
t.deepEqual(columns, expectedColumns);
140+
});
141+
142+
test('should allow direct overrides', (t) => {
143+
const expectedColumns: Column[] = [
144+
createColumn({
145+
name: 'id',
146+
type: new Utf8(),
147+
notNull: true,
148+
unique: true,
149+
}),
150+
createColumn({
151+
name: 'string',
152+
type: new Utf8(),
153+
}),
154+
];
155+
156+
const columns = objectToColumns(
157+
{
158+
id: 'the-id',
159+
string: 'test',
160+
},
161+
{
162+
overrides: new Map<string, Column>([
163+
[
164+
'id',
165+
createColumn({
166+
name: 'id',
167+
type: new Utf8(),
168+
notNull: true,
169+
unique: true,
170+
}),
171+
],
172+
]),
173+
},
174+
);
175+
t.deepEqual(columns, expectedColumns);
176+
});

‎src/transformers/transform.ts‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
import { DataType, Field, Utf8, Int64, Float64, Bool, List } from '@apache-arrow/esnext-esm';
2+
3+
import { Column, createColumn } from '../schema/column.js';
4+
import { JSONType } from '../types/json.js';
5+
6+
function defaultGetTypeFromValue(key: string, value: unknown): DataType | null {
7+
switch (typeof value) {
8+
case 'string': {
9+
return new Utf8();
10+
}
11+
case 'number': {
12+
return value % 1 === 0 ? new Int64() : new Float64();
13+
}
14+
case 'boolean': {
15+
return new Bool();
16+
}
17+
case 'object': {
18+
if (Array.isArray(value)) {
19+
if (value.length === 0) return new JSONType(); // Empty array, can't infer type
20+
21+
// Assuming array of same type, getting type of first element
22+
const elementType = defaultGetTypeFromValue(key + '.element', value[0]);
23+
if (elementType === null) return new JSONType();
24+
25+
const field = new Field('element', elementType); // 'element' can be any name as it's just for internal representation
26+
return new List(field);
27+
} else {
28+
return new JSONType();
29+
}
30+
}
31+
default: {
32+
throw new Error(`Unsupported type: ${typeof value}`);
33+
}
34+
}
35+
}
36+
37+
type Options = {
38+
skipColumns?: string[];
39+
primaryKeys?: string[];
40+
getTypeFromValue?: (key: string, value: unknown) => DataType | null | undefined;
41+
overrides?: Map<string, Column>;
42+
};
43+
44+
export function objectToColumns(
45+
object: Record<string, unknown>,
46+
{
47+
skipColumns = [],
48+
primaryKeys = [],
49+
getTypeFromValue = defaultGetTypeFromValue,
50+
overrides = new Map(),
51+
}: Options = {},
52+
): Column[] {
53+
return Object.entries(object)
54+
.filter(([key]) => !skipColumns.includes(key))
55+
.map(([key, value]): Column | null => {
56+
if (overrides.has(key)) {
57+
return overrides.get(key)!;
58+
}
59+
60+
let type = getTypeFromValue(key, value);
61+
if (type === undefined && getTypeFromValue !== defaultGetTypeFromValue) {
62+
type = defaultGetTypeFromValue(key, value);
63+
}
64+
if (type === null || type === undefined) return null;
65+
66+
const isPrimaryKey = primaryKeys.includes(key);
67+
68+
return createColumn({ name: key, type, primaryKey: isPrimaryKey });
69+
})
70+
.filter((column): column is Column => column !== null); // This is a type-guard
71+
}

0 commit comments

Comments
 (0)