Skip to content

Commit e96adb3

Browse files
disqdisq
authored
feat: Transformers (#25)
Co-authored-by: Kemal Hadimli <disq@users.noreply.github.com>
1 parent e138e40 commit e96adb3

File tree

8 files changed

+270
-0
lines changed

8 files changed

+270
-0
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,3 +128,6 @@ dist
128128
.yarn/build-state.yml
129129
.yarn/install-state.gz
130130
.pnp.*
131+
132+
# Editor settings
133+
.idea

‎README.md‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,16 @@ npm run build
3131
```bash
3232
npm test
3333
```
34+
35+
### Formatting and Linting
36+
37+
```bash
38+
# This is just to check if the code is formatted
39+
npm run format:check
40+
41+
# Automatically format code
42+
npm run format
43+
44+
# Lint
45+
npm run lint
46+
```

‎src/schema/arrow.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
export const METADATA_UNIQUE = 'cq:extension:unique';
2+
export const METADATA_PRIMARY_KEY = 'cq:extension:primary_key';
3+
export const METADATA_CONSTRAINT_NAME = 'cq:extension:constraint_name';
4+
export const METADATA_INCREMENTAL = 'cq:extension:incremental';
5+
6+
export const METADATA_TRUE = 'true';
7+
export const METADATA_FALSE = 'false';
8+
export const METADATA_TABLE_NAME = 'cq:table_name';
9+
export const METADATA_TABLE_DESCRIPTION = 'cq:table_description';

‎src/schema/column.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import { DataType, Field } from '@apache-arrow/esnext-esm';
2+
3+
import * as arrow from './arrow.js';
4+
5+
export class Column {
6+
name: string;
7+
type: DataType;
8+
description: string;
9+
primary_key: boolean;
10+
not_null: boolean;
11+
incremental_key: boolean;
12+
unique: boolean;
13+
14+
constructor(
15+
name: string,
16+
type: DataType,
17+
description: string = '',
18+
primary_key: boolean = false,
19+
not_null: boolean = false,
20+
incremental_key: boolean = false,
21+
unique: boolean = false,
22+
) {
23+
this.name = name;
24+
this.type = type;
25+
this.description = description;
26+
this.primary_key = primary_key;
27+
this.not_null = not_null;
28+
this.incremental_key = incremental_key;
29+
this.unique = unique;
30+
}
31+
32+
toString(): string {
33+
return `Column(name=${this.name}, type=${this.type}, description=${this.description}, primary_key=${this.primary_key}, not_null=${this.not_null}, incremental_key=${this.incremental_key}, unique=${this.unique})`;
34+
}
35+
36+
// JavaScript (and TypeScript) uses a single method for both string representation and debugging output
37+
toJSON(): string {
38+
return this.toString();
39+
}
40+
41+
equals(value: object): boolean {
42+
if (value instanceof Column) {
43+
return (
44+
this.name === value.name &&
45+
this.type === value.type &&
46+
this.description === value.description &&
47+
this.primary_key === value.primary_key &&
48+
this.not_null === value.not_null &&
49+
this.incremental_key === value.incremental_key &&
50+
this.unique === value.unique
51+
);
52+
}
53+
return false;
54+
}
55+
56+
toArrowField(): Field {
57+
const metadataMap = new Map<string, string>();
58+
metadataMap.set(arrow.METADATA_PRIMARY_KEY, this.primary_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE);
59+
metadataMap.set(arrow.METADATA_UNIQUE, this.unique ? arrow.METADATA_TRUE : arrow.METADATA_FALSE);
60+
metadataMap.set(arrow.METADATA_INCREMENTAL, this.incremental_key ? arrow.METADATA_TRUE : arrow.METADATA_FALSE);
61+
62+
return new Field(this.name, this.type, /*nullable=*/ !this.not_null, metadataMap);
63+
}
64+
65+
static fromArrowField(field: Field): Column {
66+
const metadata = field.metadata;
67+
const primary_key = metadata.get(arrow.METADATA_PRIMARY_KEY) === arrow.METADATA_TRUE;
68+
const unique = metadata.get(arrow.METADATA_UNIQUE) === arrow.METADATA_TRUE;
69+
const incremental_key = metadata.get(arrow.METADATA_INCREMENTAL) === arrow.METADATA_TRUE;
70+
71+
return new Column(field.name, field.type, '', primary_key, !field.nullable, unique, incremental_key);
72+
}
73+
}

‎src/transformers/openapi.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { DataType, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm';
2+
import test from 'ava';
3+
4+
import { Column } from '../schema/column.js';
5+
import { JSONType } from '../types/json.js';
6+
7+
import { oapiDefinitionToColumns } from './openapi.js';
8+
9+
const OAPI_SPEC = {
10+
swagger: '2.0',
11+
info: {
12+
version: '2.0',
13+
title: 'Test API',
14+
description: 'Unit tests APIs',
15+
},
16+
host: 'cloudquery.io',
17+
schemes: ['https'],
18+
consumes: ['application/json'],
19+
produces: ['application/json'],
20+
paths: {},
21+
definitions: {
22+
TestDefinition: {
23+
type: 'object',
24+
properties: {
25+
string: {
26+
type: 'string',
27+
},
28+
number: {
29+
type: 'number',
30+
},
31+
integer: {
32+
type: 'integer',
33+
},
34+
boolean: {
35+
type: 'boolean',
36+
},
37+
object: {
38+
$ref: '#/definitions/SomeDefinition',
39+
},
40+
array: {
41+
type: 'array',
42+
items: { $ref: '#/definitions/SomeDefinition' },
43+
},
44+
},
45+
},
46+
},
47+
};
48+
49+
test('should parse spec as expected', (t) => {
50+
const expectedColumns: Column[] = [
51+
new Column('string', new Utf8(), ''),
52+
new Column('number', new Int64(), ''),
53+
new Column('integer', new Int64(), ''),
54+
new Column('boolean', new Bool(), ''),
55+
new Column('object', new JSONType(), ''),
56+
new Column('array', new JSONType(), ''),
57+
];
58+
59+
const columns = oapiDefinitionToColumns(OAPI_SPEC['definitions']['TestDefinition']);
60+
t.deepEqual(columns, expectedColumns);
61+
});

‎src/transformers/openapi.ts‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { DataType, Field, Utf8, Int64, Bool } from '@apache-arrow/esnext-esm';
2+
3+
import { Column } from '../schema/column.js';
4+
import { JSONType } from '../types/json.js';
5+
6+
interface OAPIProperty {
7+
type?: string;
8+
description?: string;
9+
$ref?: string;
10+
items?: {
11+
$ref: string;
12+
};
13+
}
14+
15+
interface OAPIDefinition {
16+
properties: {
17+
[key: string]: OAPIProperty;
18+
};
19+
}
20+
21+
function oapiTypeToArrowType(field: OAPIProperty): DataType {
22+
const oapiType = field.type;
23+
switch (oapiType) {
24+
case 'string': {
25+
return new Utf8();
26+
}
27+
case 'number':
28+
case 'integer': {
29+
return new Int64();
30+
}
31+
case 'boolean': {
32+
return new Bool();
33+
}
34+
case 'array':
35+
case 'object': {
36+
return new JSONType();
37+
}
38+
default: {
39+
return !oapiType && '$ref' in field ? new JSONType() : new Utf8();
40+
}
41+
}
42+
}
43+
44+
export function getColumnByName(columns: Column[], name: string): Column | undefined {
45+
for (const column of columns) {
46+
if (column.name === name) {
47+
return column;
48+
}
49+
}
50+
return undefined;
51+
}
52+
53+
export function oapiDefinitionToColumns(definition: OAPIDefinition, overrideColumns: Column[] = []): Column[] {
54+
const columns: Column[] = [];
55+
for (const key in definition.properties) {
56+
const value = definition.properties[key];
57+
const columnType = oapiTypeToArrowType(value);
58+
const column = new Column(key, columnType, value.description);
59+
const overrideColumn = getColumnByName(overrideColumns, key);
60+
if (overrideColumn) {
61+
column.type = overrideColumn.type;
62+
column.primary_key = overrideColumn.primary_key;
63+
column.unique = overrideColumn.unique;
64+
}
65+
columns.push(column);
66+
}
67+
return columns;
68+
}

‎src/types/json.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import { DataType, Binary, Type } from '@apache-arrow/esnext-esm';
2+
3+
export class JSONType extends DataType<Type.Binary> {
4+
readonly extensionName: string = 'json';
5+
6+
constructor() {
7+
super();
8+
// Assuming there's no direct way to set the storage type in the constructor,
9+
// this is just a representation of the JSONType.
10+
}
11+
12+
serialize(): ArrayBuffer {
13+
// Implement your serialization logic here.
14+
return new TextEncoder().encode('json-serialized').buffer;
15+
}
16+
17+
static deserialize(storageType: Binary, serialized: ArrayBuffer): JSONType {
18+
// Implement your deserialization logic here.
19+
return new JSONType();
20+
}
21+
}

‎src/types/uuid.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import { DataType, Binary, Type } from '@apache-arrow/esnext-esm';
2+
3+
export class UUIDType extends DataType<Type.Binary> {
4+
readonly extensionName: string = 'uuid';
5+
6+
constructor() {
7+
super();
8+
// The underlying storage type is a binary of 16 bytes, representing a UUID.
9+
// Assuming there's no direct way to set the storage type in the constructor,
10+
// this is just a representation of the UUIDType.
11+
}
12+
13+
serialize(): ArrayBuffer {
14+
// Implement your serialization logic here.
15+
return new TextEncoder().encode('uuid-serialized').buffer;
16+
}
17+
18+
static deserialize(storageType: Binary, serialized: ArrayBuffer): UUIDType {
19+
// Implement your deserialization logic here.
20+
return new UUIDType();
21+
}
22+
}

0 commit comments

Comments
 (0)