Skip to content

Commit 0031166

Browse files
timstims
committed
Add ingestion module
1 parent 7827f72 commit 0031166

170 files changed

Lines changed: 11476 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎ingestion/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
example/output

‎ingestion/example/core_specs/entity/product.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"name": "product",
3+
"description": "This entity capture features for products, keys for this entity are product ids",
4+
"tags": []
5+
}

‎ingestion/example/core_specs/entity/user.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"name": "user",
3+
"description": "This entity capture features for users, keys for this entity are user ids",
4+
"tags": []
5+
}

‎ingestion/example/core_specs/feature/product.day.completed_orders.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"id": "product.day.completed_orders",
3+
"entity": "product",
4+
"granularity": "DAY",
5+
"name": "completed_orders",
6+
"owner": "feast@example.com",
7+
"description": "This feature represents a product's completed orders per day",
8+
"uri": "https://example.com/",
9+
"valueType": "INT32",
10+
"tags": [],
11+
"options": {},
12+
"dataStores": {
13+
"serving": {
14+
"id": "example_serving"
15+
},
16+
"warehouse": {
17+
"id": "example_warehouse"
18+
}
19+
}
20+
}

‎ingestion/example/core_specs/feature/user.none.age.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"id": "user.none.age",
3+
"entity": "user",
4+
"granularity": "NONE",
5+
"name": "age",
6+
"owner": "feast@example.com",
7+
"description": "This feature represents a user's age",
8+
"uri": "https://example.com/",
9+
"valueType": "INT32",
10+
"tags": [],
11+
"options": {},
12+
"dataStores": {
13+
"serving": {
14+
"id": "example_serving"
15+
},
16+
"warehouse": {
17+
"id": "example_warehouse"
18+
}
19+
}
20+
}

‎ingestion/example/core_specs/feature/user.none.completed_orders.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"id": "user.none.completed_orders",
3+
"entity": "user",
4+
"granularity": "NONE",
5+
"name": "completed_orders",
6+
"owner": "feast@example.com",
7+
"description": "This feature represents a user's total completed orders",
8+
"uri": "https://example.com/",
9+
"valueType": "INT32",
10+
"tags": [],
11+
"options": {},
12+
"dataStores": {
13+
"serving": {
14+
"id": "example_serving"
15+
},
16+
"warehouse": {
17+
"id": "example_warehouse"
18+
}
19+
}
20+
}

‎ingestion/example/core_specs/storage/example_errors.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"id": "example_errors",
3+
"type": "file.json",
4+
"options": {
5+
"path": "output/errors/data"
6+
}
7+
}

‎ingestion/example/core_specs/storage/example_serving.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"id": "example_serving",
3+
"type": "file.json",
4+
"options": {
5+
"path": "output/serving/data"
6+
}
7+
}

‎ingestion/example/core_specs/storage/example_warehouse.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"id": "example_warehouse",
3+
"type": "file.json",
4+
"options": {
5+
"path": "output/warehouse/data"
6+
}
7+
}

‎ingestion/example/import_products.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
type: file
3+
options:
4+
format: csv
5+
path: sample_data/daily_products.csv
6+
entities:
7+
- product
8+
schema:
9+
entityIdColumn: id
10+
timestampColumn: timestamp
11+
fields:
12+
- name: id
13+
- name: timestamp
14+
- featureId: product.day.completed_orders
15+

0 commit comments

Comments
 (0)