diff --git a/.github/auto_assign.yml b/.github/auto_assign.yml index 18151f7454..2a6cff517c 100644 --- a/.github/auto_assign.yml +++ b/.github/auto_assign.yml @@ -9,7 +9,6 @@ assignees: - woop - tsotnet - achals - - adchia - felixwang9817 # A number of assignees to add to the pull request diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a01bae4068..b0d3e3cb39 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -36,8 +36,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: '18.x' - registry-url: 'https://registry.npmjs.org' + node-version: "lts/*" - name: Release (Dry Run) id: get_versions run: | @@ -60,10 +59,10 @@ jobs: NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - name: Setup Node.js + uses: actions/setup-node@v3 with: - node-version: '18.x' - registry-url: 'https://registry.npmjs.org' + node-version: "lts/*" - name: Bump file versions run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION} - name: Install yarn dependencies @@ -101,10 +100,10 @@ jobs: NEXT_VERSION: ${{ needs.get_dry_release_versions.outputs.next_version }} steps: - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - name: Setup Node.js + uses: actions/setup-node@v3 with: - node-version: '18.x' - registry-url: 'https://registry.npmjs.org' + node-version: "lts/*" - name: Bump file versions (temporarily for Web UI publish) run: python ./infra/scripts/release/bump_file_versions.py ${CURRENT_VERSION} ${NEXT_VERSION} - name: Install yarn dependencies @@ -139,8 +138,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v3 with: - node-version: '18.x' - registry-url: 'https://registry.npmjs.org' + node-version: "lts/*" - name: Set up Homebrew id: set-up-homebrew uses: Homebrew/actions/setup-homebrew@master diff --git a/CHANGELOG.md b/CHANGELOG.md index f6e5a430f0..26b8baa963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,42 @@ # Changelog +# [0.35.0](https://github.com/feast-dev/feast/compare/v0.34.0...v0.35.0) (2024-01-13) + + +### Bug Fixes + +* Add async refresh to prevent synchronous refresh in main thread ([#3812](https://github.com/feast-dev/feast/issues/3812)) ([9583ed6](https://github.com/feast-dev/feast/commit/9583ed6b4ae8d3b97934bf0c80ecb236ed1e2895)) +* Adopt connection pooling for HBase ([#3793](https://github.com/feast-dev/feast/issues/3793)) ([b3852bf](https://github.com/feast-dev/feast/commit/b3852bfb8b27bf07736935f465da3067fcbac0ae)) +* Bytewax engine create configmap from object ([#3821](https://github.com/feast-dev/feast/issues/3821)) ([25e9775](https://github.com/feast-dev/feast/commit/25e97756adedfd1227d591ae74bdf60655f9067e)) +* Fix warnings from deprecated paths and update default log level ([#3757](https://github.com/feast-dev/feast/issues/3757)) ([68a8737](https://github.com/feast-dev/feast/commit/68a87379c42567f338d86cb2be90520cc6d4bfb6)) +* improve parsing bytewax job status ([5983f40](https://github.com/feast-dev/feast/commit/5983f40f8f5df5dbbcd2640f83ef82c19cdb4d19)) +* make bytewax settings unexposed ([ae1bb8b](https://github.com/feast-dev/feast/commit/ae1bb8bdd1b9e293809519971935c93c2214d791)) +* Make generated temp table name escaped ([#3797](https://github.com/feast-dev/feast/issues/3797)) ([175d796](https://github.com/feast-dev/feast/commit/175d7969b1f75ab797aff9c92a70d845297444ad)) +* Pin numpy version to avoid spammy deprecation messages ([774ed33](https://github.com/feast-dev/feast/commit/774ed33a067bf9bf087520325b72f4f4d194106a)) +* Redundant feature materialization and premature incremental materialization timestamp updates ([#3789](https://github.com/feast-dev/feast/issues/3789)) ([417b16b](https://github.com/feast-dev/feast/commit/417b16b57af7b38fbd0708b9a0c5d5035ed021fd)), closes [#6](https://github.com/feast-dev/feast/issues/6) [#7](https://github.com/feast-dev/feast/issues/7) +* Resolve hbase hotspot issue when materializing ([#3790](https://github.com/feast-dev/feast/issues/3790)) ([7376db8](https://github.com/feast-dev/feast/commit/7376db8dbd1d3168a1262fbbc0ce3899be8d0c34)) +* Set keepalives_idle None by default ([#3756](https://github.com/feast-dev/feast/issues/3756)) ([8717e9b](https://github.com/feast-dev/feast/commit/8717e9bf0fd253454982b9c9e9527c4d41906e9c)) +* Set upper bound for bigquery client due to its breaking changes ([2151c39](https://github.com/feast-dev/feast/commit/2151c39d1a8d8eba114306411dd4bd91ac0ce3f6)) +* UI project cannot handle fallback routes ([#3766](https://github.com/feast-dev/feast/issues/3766)) ([96ece0f](https://github.com/feast-dev/feast/commit/96ece0fe94a07cc6f1dabf5d6c9b061b48b06d67)) +* update dependencies versions due to conflicts ([5dc0b24](https://github.com/feast-dev/feast/commit/5dc0b241ec68aa10fd783569bf0ae12c5752f20f)) +* Update jackson and remove unnecessary logging ([#3809](https://github.com/feast-dev/feast/issues/3809)) ([018d0ea](https://github.com/feast-dev/feast/commit/018d0eab69dde63266f2c56813045ea5c5523f76)) +* upgrade the pyarrow to latest v14.0.1 for CVE-2023-47248. ([052182b](https://github.com/feast-dev/feast/commit/052182bcca046e35456674fc7d524825882f4b35)) + + +### Features + +* Add get online feature rpc to gprc server ([#3815](https://github.com/feast-dev/feast/issues/3815)) ([01db8cc](https://github.com/feast-dev/feast/commit/01db8cce6f82d4c6e496041351fb6b56eb2645b0)) +* Add materialize and materialize-incremental rest endpoints ([#3761](https://github.com/feast-dev/feast/issues/3761)) ([fa600fe](https://github.com/feast-dev/feast/commit/fa600fe3c4b1d5fdd383a9367511ac5616ee7a32)), closes [#3760](https://github.com/feast-dev/feast/issues/3760) +* add redis sentinel support ([3387a15](https://github.com/feast-dev/feast/commit/3387a15d2b7e8dea430a271570be5a19b32bd3fe)) +* add redis sentinel support ([4337c89](https://github.com/feast-dev/feast/commit/4337c89083a3cfca21ee1beef473fda13b0e9014)) +* add redis sentinel support format lint ([aad8718](https://github.com/feast-dev/feast/commit/aad8718d24d893b3ff8c5864c5b8d210cfcdb22f)) +* Add support for `table_create_disposition` in bigquery job for offline store ([#3762](https://github.com/feast-dev/feast/issues/3762)) ([6a728fe](https://github.com/feast-dev/feast/commit/6a728fe66db0286ea10301d1fe693d6dcba4e4f4)) +* Add support for in_cluster config and additional labels for bytewax materialization ([#3754](https://github.com/feast-dev/feast/issues/3754)) ([2192e65](https://github.com/feast-dev/feast/commit/2192e6527fa10f1580e4dd8f350e05e45af981b7)) +* Apply cache to load proto registry for performance ([#3702](https://github.com/feast-dev/feast/issues/3702)) ([709c709](https://github.com/feast-dev/feast/commit/709c7098dc28a35dd488f5079d3787cf1f74ec03)) +* Make bytewax job write as mini-batches ([#3777](https://github.com/feast-dev/feast/issues/3777)) ([9b0e5ce](https://github.com/feast-dev/feast/commit/9b0e5ce2d1b617fcdcf0699c8b0cf8549a5e5ac5)) +* Optimize bytewax pod resource with zero-copy ([9cf9d96](https://github.com/feast-dev/feast/commit/9cf9d965a5566a87bb7419f2e8509666076f035f)) +* Support GCS filesystem for bytewax engine ([#3774](https://github.com/feast-dev/feast/issues/3774)) ([fb6b807](https://github.com/feast-dev/feast/commit/fb6b807f8b32776d388757ca431d290c03170c66)) + # [0.34.0](https://github.com/feast-dev/feast/compare/v0.33.0...v0.34.0) (2023-09-07) diff --git a/CODEOWNERS b/CODEOWNERS index 259c13ea3f..bb154a7148 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,16 +2,21 @@ # for more info about CODEOWNERS file # Core Interfaces -/sdk/python/feast/infra/offline_stores/offline_store.py @feast-dev/maintainers @chhabrakadabra @mavysavydav @sfc-gh-madkins +/sdk/python/feast/infra/offline_stores/offline_store.py @feast-dev/maintainers @sfc-gh-madkins /sdk/python/feast/infra/online_stores/online_store.py @feast-dev/maintainers @DvirDukhan /sdk/python/feast/infra/materialization_engine/batch_materialization_engine.py @feast-dev/maintainers @whoahbot @sfc-gh-madkins # ==== Offline Stores ==== # Core utils -/sdk/python/feast/infra/offline_stores/offline_utils.py @feast-dev/maintainers @chhabrakadabra @mavysavydav @sfc-gh-madkins +/sdk/python/feast/infra/offline_stores/offline_utils.py @feast-dev/maintainers @sfc-gh-madkins + +# Offline interfaces +/sdk/python/feast/infra/offline_stores/offline_store.py @feast-dev/maintainers # BigQuery -/sdk/python/feast/infra/offline_stores/offline_store.py @feast-dev/maintainers @chhabrakadabra @mavysavydav +/sdk/python/feast/infra/offline_stores/bigquery.py @sudohainguyen +/sdk/python/feast/infra/offline_stores/bigquery_source.py @sudohainguyen +/sdk/python/tests/integration/feature_repos/universal/data_sources/bigquery.py @sudohainguyen # Snowflake /sdk/python/feast/infra/offline_stores/snowflake* @sfc-gh-madkins @@ -27,6 +32,10 @@ # ==== Online Stores ==== +# HBase +/sdk/python/feast/infra/online_stores/hbase.py @sudohainguyen +/sdk/python/feast/infra/online_stores/contrib/hbase_online_store @sudohainguyen + # Redis /sdk/python/feast/infra/online_stores/redis.py @DvirDukhan /java/feast/serving/connectors/redis/ @DvirDukhan @@ -47,8 +56,3 @@ # AWS Lambda /sdk/python/feast/infra/materialization/contrib/aws_lambda/ @achals - -# ==== Web UI ==== -/ui/ @adchia -/sdk/python/feast/ui/ @adchia -/sdk/python/feast/ui_server.py @adchia diff --git a/README.md b/README.md index f2c9348b1c..6a851d0d41 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Feast allows ML platform teams to: * **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training. * **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another. -Please see our [documentation](https://docs.feast.dev/) for more information about the project, or sign up for an [email newsletter](https://feast.dev/). +Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture ![](docs/assets/feast_marchitecture.png) @@ -145,7 +145,6 @@ pprint(feature_vector) The list below contains the functionality that contributors are planning to develop for Feast. * We welcome contribution to all items in the roadmap! -* Have questions about the roadmap? Go to the Slack channel to ask on #feast-development. * **Data Sources** * [x] [Snowflake source](https://docs.feast.dev/reference/data-sources/snowflake) @@ -214,7 +213,6 @@ Please refer to the official documentation at [Documentation](https://docs.feast * [Tutorials](https://docs.feast.dev/tutorials/tutorials-overview) * [Running Feast with Snowflake/GCP/AWS](https://docs.feast.dev/how-to-guides/feast-snowflake-gcp-aws) * [Change Log](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md) - * [Slack (#Feast)](https://slack.feast.dev/) ## 👋 Contributing Feast is a community project and is still under active development. Please have a look at our contributing and development guides if you want to contribute to the project: diff --git a/community/governance.excalidraw b/community/governance.excalidraw deleted file mode 100644 index f4c8dad9a4..0000000000 --- a/community/governance.excalidraw +++ /dev/null @@ -1,913 +0,0 @@ -{ - "type": "excalidraw", - "version": 2, - "source": "https://excalidraw.com", - "elements": [ - { - "type": "rectangle", - "version": 620, - "versionNonce": 853777363, - "isDeleted": false, - "id": "pr0yIJcUDXb4nFgowH9_r", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "dashed", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 409.5, - "y": 620.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 194, - "height": 83, - "seed": 1695250557, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [ - { - "id": "YfmPferxgVKoP70zGfYDK", - "type": "text" - }, - { - "id": "YfmPferxgVKoP70zGfYDK", - "type": "text" - }, - { - "type": "text", - "id": "YfmPferxgVKoP70zGfYDK" - }, - { - "id": "IsihlXUGDSklv2RsxX6wO", - "type": "arrow" - }, - { - "id": "G5s2AUFJ730fyPsIbA8xP", - "type": "arrow" - }, - { - "id": "j9ZVC3ZgHTsAGe3hJQecp", - "type": "arrow" - } - ], - "updated": 1662582134601, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 623, - "versionNonce": 328400605, - "isDeleted": false, - "id": "YfmPferxgVKoP70zGfYDK", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 414.5, - "y": 649.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 184, - "height": 25, - "seed": 1575229907, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582134601, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "CODEOWNERS", - "baseline": 18, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "pr0yIJcUDXb4nFgowH9_r", - "originalText": "CODEOWNERS" - }, - { - "type": "rectangle", - "version": 756, - "versionNonce": 1648798067, - "isDeleted": false, - "id": "XDy4VWWtJ9sd6hzPJDdFe", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 409.5, - "y": 779.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 194, - "height": 83, - "seed": 1925179667, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [ - { - "id": "gUz4p_oPytb5-ejbYb81N", - "type": "text" - }, - { - "id": "gUz4p_oPytb5-ejbYb81N", - "type": "text" - }, - { - "id": "gUz4p_oPytb5-ejbYb81N", - "type": "text" - }, - { - "type": "text", - "id": "gUz4p_oPytb5-ejbYb81N" - }, - { - "id": "G5s2AUFJ730fyPsIbA8xP", - "type": "arrow" - } - ], - "updated": 1662582134601, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 781, - "versionNonce": 1240013629, - "isDeleted": false, - "id": "gUz4p_oPytb5-ejbYb81N", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 414.5, - "y": 808.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 184, - "height": 25, - "seed": 140322205, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582134601, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "Contributors", - "baseline": 18, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "XDy4VWWtJ9sd6hzPJDdFe", - "originalText": "Contributors" - }, - { - "type": "text", - "version": 463, - "versionNonce": 2109720179, - "isDeleted": false, - "id": "AYJKq2RGJrSIpbfiJOf_4", - "fillStyle": "hachure", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 526, - "y": 517.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 274, - "height": 75, - "seed": 1616513981, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582134602, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "1. organize contributors\n2. influence roadmap\n3. own direction of an area", - "baseline": 68, - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "1. organize contributors\n2. influence roadmap\n3. own direction of an area" - }, - { - "type": "rectangle", - "version": 776, - "versionNonce": 519656573, - "isDeleted": false, - "id": "z5LT5d710gSTA9DjwiL3O", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 70, - "angle": 0, - "x": 1013.7117834394903, - "y": 187.5000000000001, - "strokeColor": "#000000", - "backgroundColor": "#4c6ef5", - "width": 132, - "height": 682, - "seed": 1424710877, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [ - { - "id": "RscqyQXicYOkFsE_zvran", - "type": "text" - }, - { - "id": "J7IG4T5j15pB3b_K0Cpd9", - "type": "arrow" - }, - { - "id": "XEohLLmfFl0L9Wi2ew5AU", - "type": "arrow" - }, - { - "id": "o3Pp-94PORhEiEauRcZW_", - "type": "arrow" - }, - { - "type": "text", - "id": "RscqyQXicYOkFsE_zvran" - }, - { - "id": "j9ZVC3ZgHTsAGe3hJQecp", - "type": "arrow" - }, - { - "id": "Klq-VJGZiolZnGuaNJ8k9", - "type": "arrow" - } - ], - "updated": 1662582138112, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 896, - "versionNonce": 1733426643, - "isDeleted": false, - "id": "RscqyQXicYOkFsE_zvran", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 1018.7117834394903, - "y": 476.0000000000001, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 122, - "height": 105, - "seed": 1202400115, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582138113, - "link": null, - "locked": false, - "fontSize": 28, - "fontFamily": 1, - "text": "Feast \nGitHub \nproject", - "baseline": 95, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "z5LT5d710gSTA9DjwiL3O", - "originalText": "Feast GitHub project" - }, - { - "id": "IsihlXUGDSklv2RsxX6wO", - "type": "arrow", - "x": 506.997671158975, - "y": 619, - "width": 0, - "height": 132, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [], - "strokeSharpness": "sharp", - "seed": 345290749, - "version": 680, - "versionNonce": 787007421, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "points": [ - [ - 0, - 0 - ], - [ - 0, - -132 - ] - ], - "lastCommittedPoint": null, - "startBinding": { - "elementId": "pr0yIJcUDXb4nFgowH9_r", - "focus": 0.005130630504896713, - "gap": 1.5 - }, - "endBinding": { - "elementId": "TBYpmrW2OsKEqbpZfEeJg", - "focus": 0.461338833375829, - "gap": 1 - }, - "startArrowhead": null, - "endArrowhead": "arrow" - }, - { - "id": "G5s2AUFJ730fyPsIbA8xP", - "type": "arrow", - "x": 506.9985864097345, - "y": 776, - "width": 0.9914493467237548, - "height": 71, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [], - "strokeSharpness": "sharp", - "seed": 241364467, - "version": 241, - "versionNonce": 649485971, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "points": [ - [ - 0, - 0 - ], - [ - -0.9914493467237548, - -71 - ] - ], - "lastCommittedPoint": null, - "startBinding": { - "elementId": "XDy4VWWtJ9sd6hzPJDdFe", - "focus": 0.011569796958606356, - "gap": 3.5 - }, - "endBinding": { - "elementId": "pr0yIJcUDXb4nFgowH9_r", - "focus": 0.011204382815075232, - "gap": 1.5 - }, - "startArrowhead": null, - "endArrowhead": "arrow" - }, - { - "id": "TBYpmrW2OsKEqbpZfEeJg", - "type": "rectangle", - "x": 409.5, - "y": 188, - "width": 361.99999999999994, - "height": 298, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "solid", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 30, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 1515491581, - "version": 231, - "versionNonce": 593345661, - "isDeleted": false, - "boundElements": [ - { - "id": "IsihlXUGDSklv2RsxX6wO", - "type": "arrow" - }, - { - "id": "Klq-VJGZiolZnGuaNJ8k9", - "type": "arrow" - } - ], - "updated": 1662582134602, - "link": null, - "locked": false - }, - { - "id": "YEEHpa4RXaR8G9YW55v25", - "type": "rectangle", - "x": 428.5, - "y": 398, - "width": 163.61445783132532, - "height": 70.00000000000001, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 932648787, - "version": 319, - "versionNonce": 398988755, - "isDeleted": false, - "boundElements": [ - { - "type": "text", - "id": "8iyUZwSph5yMVrXehf6vg" - }, - { - "id": "o3Pp-94PORhEiEauRcZW_", - "type": "arrow" - }, - { - "id": "IsihlXUGDSklv2RsxX6wO", - "type": "arrow" - } - ], - "updated": 1662582134602, - "link": null, - "locked": false - }, - { - "id": "8iyUZwSph5yMVrXehf6vg", - "type": "text", - "x": 433.5, - "y": 422.5, - "width": 153.61445783132532, - "height": 21, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 1803538003, - "version": 365, - "versionNonce": 952837341, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "text": "Area maintainers", - "fontSize": 16.697223677317968, - "fontFamily": 1, - "textAlign": "center", - "verticalAlign": "middle", - "baseline": 15, - "containerId": "YEEHpa4RXaR8G9YW55v25", - "originalText": "Area maintainers" - }, - { - "type": "rectangle", - "version": 355, - "versionNonce": 1753998195, - "isDeleted": false, - "id": "Wh-PpzmGy1bWJko0akD-a", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 429.8072289156627, - "y": 257.1185567010309, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 161, - "height": 68.88144329896907, - "seed": 1844448573, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "boundElements": [ - { - "id": "OJCS1hAx71BD6u1jesJzR", - "type": "text" - }, - { - "type": "text", - "id": "OJCS1hAx71BD6u1jesJzR" - }, - { - "id": "o3Pp-94PORhEiEauRcZW_", - "type": "arrow" - } - ], - "updated": 1662582134602, - "link": null, - "locked": false - }, - { - "type": "text", - "version": 409, - "versionNonce": 556564797, - "isDeleted": false, - "id": "OJCS1hAx71BD6u1jesJzR", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 434.8072289156627, - "y": 271.55927835051546, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 151, - "height": 40, - "seed": 852504851, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582134602, - "link": null, - "locked": false, - "fontSize": 16.413043478260864, - "fontFamily": 1, - "text": "Project \nmaintainers", - "baseline": 34, - "textAlign": "center", - "verticalAlign": "middle", - "containerId": "Wh-PpzmGy1bWJko0akD-a", - "originalText": "Project maintainers" - }, - { - "id": "o3Pp-94PORhEiEauRcZW_", - "type": "arrow", - "x": 510.1952932956257, - "y": 396.60017389144207, - "width": 0.34508644012566947, - "height": 69.20034778288408, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "fillStyle": "hachure", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 1889236627, - "version": 572, - "versionNonce": 918879507, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "points": [ - [ - 0, - 0 - ], - [ - 0.34508644012566947, - -69.20034778288408 - ] - ], - "lastCommittedPoint": null, - "startBinding": { - "elementId": "YEEHpa4RXaR8G9YW55v25", - "focus": -0.0035794947090358044, - "gap": 1.3998261085579315 - }, - "endBinding": { - "elementId": "Wh-PpzmGy1bWJko0akD-a", - "focus": -0.0051056226396315905, - "gap": 1.3998261085579884 - }, - "startArrowhead": null, - "endArrowhead": "arrow" - }, - { - "id": "4CHi-UfB3oI1PAfcFm2o_", - "type": "text", - "x": 528.5, - "y": 354.5, - "width": 218, - "height": 20, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "fillStyle": "hachure", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 2054408115, - "version": 238, - "versionNonce": 1105416605, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "text": "break ties by majority vote", - "fontSize": 16, - "fontFamily": 1, - "textAlign": "left", - "verticalAlign": "top", - "baseline": 14, - "containerId": null, - "originalText": "break ties by majority vote" - }, - { - "id": "gHvMhIQl4S1SxPE8kzHLx", - "type": "text", - "x": 431.8072289156627, - "y": 201.5, - "width": 157, - "height": 35, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "solid", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [ - "mcHoJ-dlfU3T8l_C93UPa" - ], - "strokeSharpness": "sharp", - "seed": 1597289651, - "version": 154, - "versionNonce": 1038738099, - "isDeleted": false, - "boundElements": null, - "updated": 1662582134602, - "link": null, - "locked": false, - "text": "Maintainers", - "fontSize": 28, - "fontFamily": 1, - "textAlign": "left", - "verticalAlign": "top", - "baseline": 25, - "containerId": null, - "originalText": "Maintainers" - }, - { - "type": "text", - "version": 545, - "versionNonce": 1478563325, - "isDeleted": false, - "id": "_qJ5MtLgnvmF1-EDKX6qg", - "fillStyle": "hachure", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 533, - "y": 732.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 106, - "height": 25, - "seed": 1870614973, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582134602, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "review PRs", - "baseline": 18, - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "review PRs" - }, - { - "id": "j9ZVC3ZgHTsAGe3hJQecp", - "type": "arrow", - "x": 610.590909090909, - "y": 673.6931323855418, - "width": 394.7818338530517, - "height": 1.1368683772161603e-13, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "solid", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [], - "strokeSharpness": "sharp", - "seed": 1115132605, - "version": 594, - "versionNonce": 1612334739, - "isDeleted": false, - "boundElements": null, - "updated": 1662582138112, - "link": null, - "locked": false, - "points": [ - [ - 0, - 0 - ], - [ - 394.7818338530517, - 1.1368683772161603e-13 - ] - ], - "lastCommittedPoint": null, - "startBinding": { - "elementId": "pr0yIJcUDXb4nFgowH9_r", - "gap": 7.0909090909090455, - "focus": 0.2817622261576348 - }, - "endBinding": { - "elementId": "z5LT5d710gSTA9DjwiL3O", - "gap": 9.339040495529549, - "focus": -0.4257863119810608 - }, - "startArrowhead": null, - "endArrowhead": "arrow" - }, - { - "id": "Klq-VJGZiolZnGuaNJ8k9", - "type": "arrow", - "x": 775.7385321100917, - "y": 334, - "width": 233.2672383568049, - "height": 0, - "angle": 0, - "strokeColor": "#000000", - "backgroundColor": "#868e96", - "fillStyle": "solid", - "strokeWidth": 1, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "groupIds": [], - "strokeSharpness": "sharp", - "seed": 632787667, - "version": 198, - "versionNonce": 1893334067, - "isDeleted": false, - "boundElements": null, - "updated": 1662582138112, - "link": null, - "locked": false, - "points": [ - [ - 0, - 0 - ], - [ - 233.2672383568049, - 0 - ] - ], - "lastCommittedPoint": null, - "startBinding": { - "elementId": "TBYpmrW2OsKEqbpZfEeJg", - "gap": 4.238532110091741, - "focus": -0.020134228187919462 - }, - "endBinding": { - "elementId": "z5LT5d710gSTA9DjwiL3O", - "gap": 5.7060129725937765, - "focus": 0.5703812316715546 - }, - "startArrowhead": null, - "endArrowhead": "arrow" - }, - { - "type": "text", - "version": 651, - "versionNonce": 1375877757, - "isDeleted": false, - "id": "diazwl57WWW_7gfm7wMea", - "fillStyle": "hachure", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 678, - "y": 637.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 262, - "height": 25, - "seed": 2077675699, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582152851, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "merge PRs if no objections", - "baseline": 18, - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "merge PRs if no objections" - }, - { - "type": "text", - "version": 658, - "versionNonce": 1558756051, - "isDeleted": false, - "id": "_T1wMHFNqfA6a8Ku2OLDl", - "fillStyle": "hachure", - "strokeWidth": 2, - "strokeStyle": "solid", - "roughness": 1, - "opacity": 100, - "angle": 0, - "x": 840, - "y": 296.5, - "strokeColor": "#000000", - "backgroundColor": "transparent", - "width": 102, - "height": 25, - "seed": 1987233139, - "groupIds": [], - "strokeSharpness": "sharp", - "boundElements": [], - "updated": 1662582148465, - "link": null, - "locked": false, - "fontSize": 20, - "fontFamily": 1, - "text": "merge PRs", - "baseline": 18, - "textAlign": "left", - "verticalAlign": "top", - "containerId": null, - "originalText": "merge PRs" - } - ], - "appState": { - "gridSize": null, - "viewBackgroundColor": "#ffffff" - }, - "files": {} -} \ No newline at end of file diff --git a/community/governance.md b/community/governance.md index 89cf800bc8..087b3599db 100644 --- a/community/governance.md +++ b/community/governance.md @@ -42,11 +42,7 @@ A formal governance structure helps us to On a high level, the key moving parts of the community are: - **GitHub activity** (issues + pull requests) -- **Slack community** ([slack.feast.dev](slack.feast.dev)) - - `#feast-development` is where design discussions happen amongst contributors - - Other Slack channels exist for users to ask and answer questions. - **RFCs** ([drive folder](https://drive.google.com/drive/u/0/folders/1msUsgmDbVBaysmhBlg9lklYLLTMk4bC3)) for detailed discussions -- **Community calls** (biweekly) to discuss best practices, contributions, and announce changes - **Maintainer syncs** (monthly) for [maintainers](maintainers.md) to discuss project direction and health With this structure, users and contributors largely self-organize and contribute changes as per [lazy consensus](#lazy-consensus). If there is active opposition and unresolvable conflict, then maintainers step in to break ties or make decisions. @@ -61,10 +57,6 @@ Anyone interested in the project can join the community to: - contribute to the project design - participate in the decision-making process. -The general decision making workflow is as follows: - - - > **Note**: There may not always a corresponding CODEOWNER for the affected code, in which case the responsibility falls on other maintainers or contributors with write access to review + merge the PR # Roles And Responsibilities @@ -96,7 +88,7 @@ In addition to their actions as users, contributors may also find themselves doi * Writing, editing, translating or reviewing the documentation * Organizing events or evangelizing the project -Contributors engage with the project through the issue tracker and slack community, or by writing or editing documentation. They submit changes to the project itself via Pull Requests (PRs), which will be considered for inclusion in the project by existing maintainers (see next section). +Contributors engage with the project through the issue tracker or by writing or editing documentation. They submit changes to the project itself via Pull Requests (PRs), which will be considered for inclusion in the project by existing maintainers (see next section). Contributors should follow the following guides when creating PRs: - [Contribution Process](https://docs.feast.dev/project/contributing) @@ -116,27 +108,14 @@ Maintainers are community members who have shown that they are committed to Feas > **Note**: maintainers, like other contributors, must make changes to Feast via pull requests (with code review). This applies to all changes to documentation, code, configuration, governance, etc. -### Types of maintainers - -There are two kinds of maintainers - -1. **Project maintainers** control overall project organization and resolving disputes. They also - - Attend a regular maintainers sync - - Participate in strategic planning, approve changes to the governance model, and manage the copyrights within the project outputs. - - (optional) Attend community calls - - (optional) Planning project roadmaps and articulating vision - - (optional) Guide design decisions to reinforce key project values (e.g. simplicity) -2. **Area maintainers** own a specific technical area (which may span code modules), often specifically targeting a user journey or tech stack. They - - are generally point people in GitHub or Slack on discussions in that area (e.g. tagged in `#feast-development`) - - (optional) help drive roadmap decisions - -> **Note:** project maintainers may also be area maintainers, but this does not give their ideas increased weight over other area maintainers. - -Decisions that need tie breakers may require intervention via project maintainers majority consensus. +Maintainers control overall project organization and resolving disputes. They also +- Attend a regular maintainers sync +- Participate in strategic planning, approve changes to the governance model, and manage the copyrights within the project outputs. +- (optional) Planning project roadmaps and articulating vision +- (optional) Guide design decisions to reinforce key project values (e.g. simplicity) ### Optional maintainer responsibilities Other optional activites a maintainer (project or area maintainer) may participate in: - * Monitor email aliases and our Slack (#feast-general, #feast-development, #feast-beginners). * Perform code reviews for other maintainers and the community. The areas of specialization listed in [OWNERS.md](OWNERS.md) can be used to help with routing an issue/question to the right person. * Triage GitHub issues, applying [labels]([https://github.com/feast-dev/feast/labels](https://github.com/feast-dev/feast/labels)) to each new item. Labels are extremely useful for future issue follow ups. Adding labels is somewhat subjective, so please use your best judgment. * Triage build issues, filing issues for known flaky builds or bugs, fixing or finding someone to fix any master build breakages. diff --git a/community/governance.png b/community/governance.png deleted file mode 100644 index c2b00930e3..0000000000 Binary files a/community/governance.png and /dev/null differ diff --git a/community/maintainers.md b/community/maintainers.md index cdf78b150c..e66dbeb762 100644 --- a/community/maintainers.md +++ b/community/maintainers.md @@ -7,36 +7,25 @@ See [Governance](governance.md) for what each maintainer type is In alphabetical order | Name | GitHub Username | Email | Organization | -| -------------- | ---------------- | --------------------------- | ------------------ | -| Abhin Chhabra | `chhabrakadabra` | chhabra.abhin@gmail.com | Shopify | +| -------------- | ---------------- |-----------------------------| ------------------ | | Achal Shah | `achals` | achals@gmail.com | Tecton | -| Danny Chiao | `adchia` | d.chiao@gmail.com | Tecton | -| David Liu | `mavysavydav` | davidyliuliu@gmail.com | Twitter | | Felix Wang | `felixwang9817` | wangfelix98@gmail.com | Tecton | | Kevin Zhang | `kevjumba` | kevin.zhang.13499@gmail.com | Tecton | -| Matt Delacour | `MattDelac` | mdelacour@hey.com | (formerly) Shopify | | Miles Adkins | `sfc-gh-madkins` | miles.adkins@snowflake.com | Snowflake | | Willem Pienaar | `woop` | will.pienaar@gmail.com | Tecton | | Zhiling Chen | `zhilingc` | chnzhlng@gmail.com | GetGround | -## Area maintainers - -Generally, with contribution questions here, default to `#feast-development` in the [slack.feast.dev](slack.feast.dev) Slack channel, but these may be folks for you to tag in messages - -| Area | Description | Name | -| -------------------- | -------------------------------------------------------------------------- | --------------------------------------------- | -| Data ingestion | ingesting batch + stream data into the online store (materialization) | Achal Shah,
Felix Wang,
Kevin Zhang | -| Developer experience | tooling, testing, documentation, tutorials | Achal Shah | -| Feature serving | optimization, caching, deployment patterns, batch retrieval, range queries | Dvir Dukhan | -| Ops | general deployment concerns, CI/CD, versioning | Keith Adler,
Danny Chiao,
Felix Wang | -| Web UI | i.e. `feast ui` output | Danny Chiao,
David Liu | - ## Emeritus Maintainers -| Name | GitHub Username | Email | Organization | -| ------------------- | --------------- | --------------------------- | ------------ | -| Oleg Avdeev | oavdeev | oleg.v.avdeev@gmail.com | Tecton | -| Oleksii Moskalenko | pyalex | moskalenko.alexey@gmail.com | Tecton | -| Jay Parthasarthy | jparthasarthy | jparthasarthy@gmail.com | Tecton | -| Pradithya Aria Pura | pradithya | pradithya.aria@gmail.com | Gojek | -| Tsotne Tabidze | tsotnet | tsotnet@gmail.com | Tecton | \ No newline at end of file +| Name | GitHub Username | Email | Organization | +|---------------------|-----------------|-----------------------------|-------------------| +| Oleg Avdeev | oavdeev | oleg.v.avdeev@gmail.com | Tecton | +| Oleksii Moskalenko | pyalex | moskalenko.alexey@gmail.com | Tecton | +| Jay Parthasarthy | jparthasarthy | jparthasarthy@gmail.com | Tecton | +| Danny Chiao | adchia | danny@tecton.ai | Tecton | +| Pradithya Aria Pura | pradithya | pradithya.aria@gmail.com | Gojek | +| Tsotne Tabidze | tsotnet | tsotnet@gmail.com | Tecton | +| Abhin Chhabra | chhabrakadabra | chhabra.abhin@gmail.com | Shopify | +| Danny Chiao | adchia | danny@tecton.ai | Tecton | +| David Liu | mavysavydav | davidyliuliu@gmail.com | Twitter | +| Matt Delacour | MattDelac | mdelacour@hey.com | Shopify | diff --git a/docs/README.md b/docs/README.md index a305c4aecd..66c7548440 100644 --- a/docs/README.md +++ b/docs/README.md @@ -57,7 +57,7 @@ Many companies have used Feast to power real-world ML use cases such as: ## How can I get started? {% hint style="info" %} -The best way to learn Feast is to use it. Join our [Slack channel](http://slack.feast.dev) and head over to our [Quickstart](getting-started/quickstart.md) and try it out! +The best way to learn Feast is to use it. Head over to our [Quickstart](getting-started/quickstart.md) and try it out! {% endhint %} Explore the following resources to get started with Feast: diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 9b22d1e286..c80ded2adf 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -90,6 +90,7 @@ * [SQLite](reference/online-stores/sqlite.md) * [Snowflake](reference/online-stores/snowflake.md) * [Redis](reference/online-stores/redis.md) + * [Dragonfly](reference/online-stores/dragonfly.md) * [Datastore](reference/online-stores/datastore.md) * [DynamoDB](reference/online-stores/dynamodb.md) * [Bigtable](reference/online-stores/bigtable.md) diff --git a/docs/community.md b/docs/community.md index 098b6b3f90..21cca702bf 100644 --- a/docs/community.md +++ b/docs/community.md @@ -4,13 +4,6 @@ * [GitHub Repository](https://github.com/feast-dev/feast/): Find the complete Feast codebase on GitHub. * [Community Governance Doc](https://github.com/feast-dev/feast/blob/master/community): See the governance model of Feast, including who the maintainers are and how decisions are made. -* [Slack](https://slack.feast.dev): Feel free to ask questions or say hello! This is the main place where maintainers and contributors brainstorm and where users ask questions or discuss best practices. - * Feast users should join `#feast-general` or `#feast-beginners` to ask questions - * Feast developers / contributors should join `#feast-development` -* [Mailing list](https://groups.google.com/d/forum/feast-dev): We have both a user and developer mailing list. - * Feast users should join [feast-discuss@googlegroups.com](mailto:feast-discuss@googlegroups.com) group by clicking [here](https://groups.google.com/g/feast-discuss). - * Feast developers / contributors should join [feast-dev@googlegroups.com](mailto:feast-dev@googlegroups.com) group by clicking [here](https://groups.google.com/d/forum/feast-dev). -* [Community Calendar](https://calendar.google.com/calendar/u/0?cid=ZTFsZHVhdGM3MDU3YTJucTBwMzNqNW5rajBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ): Includes community calls and design meetings. * [Google Folder](https://drive.google.com/drive/u/0/folders/1jgMHOPDT2DvBlJeO9LCM79DP4lm4eOrR): This folder is used as a central repository for all Feast resources. For example: * Design proposals in the form of Request for Comments (RFC). * User surveys and meeting minutes. @@ -19,36 +12,4 @@ ## How can I get help? -* **Slack:** Need to speak to a human? Come ask a question in our Slack channel (link above). * **GitHub Issues:** Found a bug or need a feature? [Create an issue on GitHub](https://github.com/feast-dev/feast/issues/new). -* **StackOverflow:** Need to ask a question on how to use Feast? We also monitor and respond to [StackOverflow](https://stackoverflow.com/questions/tagged/feast). - -## Community Calls - -### General community call (biweekly) -We have a user and contributor community call every two weeks (US & EU friendly). - -{% hint style="info" %} -Please join the above Feast user groups in order to see calendar invites to the community calls -{% endhint %} - -#### Frequency (every 2 weeks) - -* Tuesday 10:00 am to 10:30 am PST - -#### Links - -* Zoom: [https://zoom.us/j/6325193230](https://zoom.us/j/6325193230) -* Meeting notes (incl recordings): [https://bit.ly/feast-notes](https://bit.ly/feast-notes) - -### Developers call (biweekly) -We also have a `#feast-development` community call every two weeks, where we discuss contributions + brainstorm best practices. - -#### Frequency (every 2 weeks) - -* Tuesday 8:00 am to 8:30 am PST - -#### Links - -* Meeting notes (incl recordings): [Feast Development Biweekly](https://docs.google.com/document/d/1zUbIWFWjaBEVlToOdupnmKQwgAtFYx41sPoEEEdd2io/edit#) -* Zoom: [https://zoom.us/j/93657748160?pwd=K3ZpdzhqejgrcXNhc3BlSjFMdzUxdz09](https://zoom.us/j/93657748160?pwd=K3ZpdzhqejgrcXNhc3BlSjFMdzUxdz09) diff --git a/docs/getting-started/faq.md b/docs/getting-started/faq.md index a511ddb0dc..9b7eb834bf 100644 --- a/docs/getting-started/faq.md +++ b/docs/getting-started/faq.md @@ -3,7 +3,7 @@ {% hint style="info" %} **Don't see your question?** -We encourage you to ask questions on [Slack](https://slack.feast.dev) or [GitHub](https://github.com/feast-dev/feast). Even better, once you get an answer, add the answer to this FAQ via a [pull request](../project/development-guide.md)! +We encourage you to ask questions on [GitHub](https://github.com/feast-dev/feast). Even better, once you get an answer, add the answer to this FAQ via a [pull request](../project/development-guide.md)! {% endhint %} ## Getting started diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index b30bdb585c..d10e8a174a 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -555,9 +555,7 @@ show up in the upcoming concepts + architecture + tutorial pages as well. ## Next steps -* Join the [email newsletter](https://feast.dev/) to get new updates on Feast / feature stores. * Read the [Concepts](concepts/) page to understand the Feast data model. * Read the [Architecture](architecture-and-components/) page. * Check out our [Tutorials](../tutorials/tutorials-overview/) section for more examples on how to use Feast. * Follow our [Running Feast with Snowflake/GCP/AWS](../how-to-guides/feast-snowflake-gcp-aws/) guide for a more in-depth tutorial on using Feast. -* Join other Feast users and contributors in [Slack](https://slack.feast.dev) and become part of the community! diff --git a/docs/project/contributing.md b/docs/project/contributing.md index 9a3e3e1a3e..cded378951 100644 --- a/docs/project/contributing.md +++ b/docs/project/contributing.md @@ -2,21 +2,15 @@ ## Getting started After familiarizing yourself with the documentation, the simplest way to get started is to: -1. Join the `#feast-development` [Slack channel](https://tectonfeast.slack.com/archives/C01NTDB88QK), where contributors discuss ideas and PRs -2. Join our Google Groups in order to get access to RFC folders + get invites to community calls. See [community](../community.md) for more details. -3. Setup your developer environment by following [development guide](development-guide.md). -4. Either create a [GitHub issue](https://github.com/feast-dev/feast/issues) or make a draft PR (following [development guide](development-guide.md)) to get the ball rolling! +1. Setup your developer environment by following [development guide](development-guide.md). +2. Either create a [GitHub issue](https://github.com/feast-dev/feast/issues) or make a draft PR (following [development guide](development-guide.md)) to get the ball rolling! ## Decision making process *See [governance](../../community/governance.md) for more details here* We follow a process of [lazy consensus](http://community.apache.org/committers/lazyConsensus.html). If you believe you know what the project needs then just start development. As long as there is no active opposition and the PR has been approved by maintainers or CODEOWNERS, contributions will be merged. -We use our `#feast-development` [Slack channel](https://tectonfeast.slack.com/archives/C01NTDB88QK), [GitHub issues](https://github.com/feast-dev/feast/issues), and [GitHub pull requests](https://github.com/feast-dev/feast/pulls) to communicate development ideas. - -The general decision making workflow is as follows: - - +We use our [GitHub issues](https://github.com/feast-dev/feast/issues), and [GitHub pull requests](https://github.com/feast-dev/feast/pulls) to communicate development ideas. > **Note**: There may not always a corresponding CODEOWNER for the affected code, in which case the responsibility falls on other maintainers or contributors with write access to review + merge the PR @@ -30,7 +24,7 @@ See also [Making a pull request](development-guide.md#making-a-pull-request) for ## Resources -- [Community](../community.md) for other ways to get involved with the community (e.g. joining community calls) +- [Community](../community.md) for other ways to get involved with the community - [Development guide](development-guide.md) for tips on how to contribute - [Feast GitHub issues](https://github.com/feast-dev/feast/issues) to see what others are working on - [Feast RFCs](https://drive.google.com/drive/u/0/folders/1msUsgmDbVBaysmhBlg9lklYLLTMk4bC3) for a folder of previously written RFCs \ No newline at end of file diff --git a/docs/project/development-guide.md b/docs/project/development-guide.md index 69a1ab298a..931d0243d2 100644 --- a/docs/project/development-guide.md +++ b/docs/project/development-guide.md @@ -51,13 +51,6 @@ The compatibility policy for Feast can be found [here](compatibility.md), and sh ## Community See [Contribution process](./contributing.md) and [Community](../community.md) for details on how to get more involved in the community. -A quick few highlights: -- [RFCs](https://drive.google.com/drive/u/0/folders/0AAe8j7ZK3sxSUk9PVA) -- [Community Slack](https://slack.feast.dev/) -- [Feast Dev Mailing List](https://groups.google.com/g/feast-dev) -- [Community Calendar](https://calendar.google.com/calendar/u/0?cid=ZTFsZHVhdGM3MDU3YTJucTBwMzNqNW5rajBAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ) - - Includes biweekly community calls at 10AM PST - ## Making a pull request We use the convention that the assignee of a PR is the person with the next action. diff --git a/docs/project/release-process.md b/docs/project/release-process.md index 0aa7d3fb5b..d3ff34bbc3 100644 --- a/docs/project/release-process.md +++ b/docs/project/release-process.md @@ -16,14 +16,14 @@ If you were cutting Feast 0.22.3, for example, you might do: After this step, you will have all the changes you need in the branch. -### 2. Pre-release verification +### 2. Pre-release verification (currently broken) A lot of things can go wrong. One of the most common is getting the wheels to build correctly (and not accidentally building dev wheels from improper tagging or local code changes during the release process). Another possible failure is that the Docker images might not build correctly. We verify the building the wheels and Docker images in **your fork** of Feast, not the main feast-dev/feast repo. -#### For minor releases (e.g. v0.22.0) +#### 2a. Verifying minor releases (e.g. v0.22.0) 1. Merge upstream master changes into your **fork**. Make sure you are running the workflow off of your fork! 2. Create a tag manually for the release on your fork. For example, if you are doing a release for version 0.22.0, create a tag by doing the following. - Checkout master branch and run `git tag v0.22.0`. @@ -37,17 +37,7 @@ We verify the building the wheels and Docker images in **your fork** of Feast, n 5. Run the workflow off of the tag you just created(`v0.22.0` in this case, **not** the master branch) and verify that the workflow worked (i.e ensure that all jobs are green). -#### For patch releases (e.g. v0.22.3) -You should already have checked out the existing minor release branch from step 1 (e.g. `v0.22-branch`). -1. Push the minor release branch to your fork (`git push -u origin `). -2. Add a patch release tag (e.g `v0.22.1`) by running `git tag `. - > This is important. If you don't have a tag, then the wheels you build will be **dev wheels**, which we can't - > push. The release process will automatically produce a tag for you via Semantic Release. -3. Push tags to your **origin branch** (not the upstream feast-dev/feast branch) with `git push origin `. -4. Kick off `build_wheels` workflow in your fork in the same way as is detailed in the last section, running the - workflow from this tag you just pushed up. - -### 3. Release for Python and Java SDK +### 2. Release for Python and Java SDK 1. Generate a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) or retrieve your saved personal access token. * The personal access token should have all of the permissions under the `repo` checkbox. 2. Access the `Actions` tab on the main `feast-dev/feast` repo and find the `release` action. @@ -81,7 +71,7 @@ so it helps to have a high level overview. See https://github.com/feast-dev/feas #### 4c: Update documentation -In the Feast Gitbook (ask [Danny Chiao](https://tectonfeast.slack.com/team/U029405HFEU) in Slack for access): +In the Feast Gitbook: 1. Create a new space within the Feast collection 2. Go to the overflow menu on the top -> Synchronize with Git 1. Specify GitHub as the provider diff --git a/docs/reference/feast-cli-commands.md b/docs/reference/feast-cli-commands.md index 38e85843d4..7bdea19e61 100644 --- a/docs/reference/feast-cli-commands.md +++ b/docs/reference/feast-cli-commands.md @@ -11,8 +11,6 @@ Usage: feast [OPTIONS] COMMAND [ARGS]... For more information, see our public docs at https://docs.feast.dev/ - For any questions, you can reach us at https://slack.feast.dev/ - Options: -c, --chdir TEXT Switch to a different feature repository directory before executing the given subcommand. diff --git a/docs/reference/online-stores/README.md b/docs/reference/online-stores/README.md index 2fdfd50f7c..f86e6f6a1d 100644 --- a/docs/reference/online-stores/README.md +++ b/docs/reference/online-stores/README.md @@ -18,6 +18,10 @@ Please see [Online Store](../../getting-started/architecture-and-components/onli [redis.md](redis.md) {% endcontent-ref %} +{% content-ref url="dragonfly.md" %} +[dragonfly.md](dragonfly.md) +{% endcontent-ref %} + {% content-ref url="datastore.md" %} [datastore.md](datastore.md) {% endcontent-ref %} diff --git a/docs/reference/online-stores/redis.md b/docs/reference/online-stores/redis.md index c08cef2a3e..ae7f8b4c5c 100644 --- a/docs/reference/online-stores/redis.md +++ b/docs/reference/online-stores/redis.md @@ -45,6 +45,21 @@ online_store: ``` {% endcode %} +Connecting to a Redis Sentinel with SSL enabled and password authentication: + +{% code title="feature_store.yaml" %} +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + redis_type: redis_sentinel + sentinel_master: mymaster + connection_string: "redis1:26379,ssl=true,password=my_password" +``` +{% endcode %} + Additionally, the redis online store also supports automatically deleting data via a TTL mechanism. The TTL is applied at the entity level, so feature values from any associated feature views for an entity are removed together. This TTL can be set in the `feature_store.yaml`, using the `key_ttl_seconds` field in the online store. For example: diff --git a/docs/roadmap.md b/docs/roadmap.md index d5dc88005b..a04ede7c99 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -3,7 +3,6 @@ The list below contains the functionality that contributors are planning to develop for Feast. * We welcome contribution to all items in the roadmap! -* Have questions about the roadmap? Go to the Slack channel to ask on #feast-development. * **Data Sources** * [x] [Snowflake source](https://docs.feast.dev/reference/data-sources/snowflake) diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index 6e07d3e23b..f84457ac02 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -1066,7 +1066,6 @@ "- Read the [Concepts](https://docs.feast.dev/getting-started/concepts/) page to understand the Feast data model and architecture.\n", "- Check out our [Tutorials](https://docs.feast.dev/tutorials/tutorials-overview) section for more examples on how to use Feast.\n", "- Follow our [Running Feast with Snowflake/GCP/AWS](https://docs.feast.dev/how-to-guides/feast-snowflake-gcp-aws) guide for a more in-depth tutorial on using Feast.\n", - "- Join other Feast users and contributors in [Slack](https://slack.feast.dev/) and become part of the community!" ] } ], diff --git a/go.mod b/go.mod index 68d47d7070..20c52d3221 100644 --- a/go.mod +++ b/go.mod @@ -37,11 +37,11 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect golang.org/x/exp v0.0.0-20220407100705-7b9b53b0aca4 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/mod v0.8.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index a2c7501e82..990ff9b1ba 100644 --- a/go.sum +++ b/go.sum @@ -895,6 +895,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -951,10 +952,10 @@ golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1017,8 +1018,10 @@ golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1059,6 +1062,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1144,13 +1148,17 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1163,8 +1171,10 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1236,10 +1246,10 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/infra/charts/feast-feature-server/Chart.yaml b/infra/charts/feast-feature-server/Chart.yaml index 6631d37784..6111639b68 100644 --- a/infra/charts/feast-feature-server/Chart.yaml +++ b/infra/charts/feast-feature-server/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: feast-feature-server description: Feast Feature Server in Go or Python type: application -version: 0.34.0 +version: 0.35.0 keywords: - machine learning - big data diff --git a/infra/charts/feast-feature-server/README.md b/infra/charts/feast-feature-server/README.md index ad88d08217..2467b60e1e 100644 --- a/infra/charts/feast-feature-server/README.md +++ b/infra/charts/feast-feature-server/README.md @@ -1,6 +1,6 @@ # Feast Python / Go Feature Server Helm Charts -Current chart version is `0.34.0` +Current chart version is `0.35.0` ## Installation @@ -30,7 +30,7 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/python-helm-d | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"feastdev/feature-server"` | Docker image for Feature Server repository | -| image.tag | string | `"0.34.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | +| image.tag | string | `"0.35.0"` | The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) | | imagePullSecrets | list | `[]` | | | livenessProbe.initialDelaySeconds | int | `30` | | | livenessProbe.periodSeconds | int | `30` | | diff --git a/infra/charts/feast-feature-server/values.yaml b/infra/charts/feast-feature-server/values.yaml index d46f1b685b..2383d6fa9f 100644 --- a/infra/charts/feast-feature-server/values.yaml +++ b/infra/charts/feast-feature-server/values.yaml @@ -9,7 +9,7 @@ image: repository: feastdev/feature-server pullPolicy: IfNotPresent # image.tag -- The Docker image tag (can be overwritten if custom feature server deps are needed for on demand transforms) - tag: 0.34.0 + tag: 0.35.0 imagePullSecrets: [] nameOverride: "" diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index e0f530e05e..c53b73848c 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v1 description: Feature store for machine learning name: feast -version: 0.34.0 +version: 0.35.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md index fff6d0261b..d8d4a3d376 100644 --- a/infra/charts/feast/README.md +++ b/infra/charts/feast/README.md @@ -8,7 +8,7 @@ This repo contains Helm charts for Feast Java components that are being installe ## Chart: Feast -Feature store for machine learning Current chart version is `0.34.0` +Feature store for machine learning Current chart version is `0.35.0` ## Installation @@ -65,8 +65,8 @@ See [here](https://github.com/feast-dev/feast/tree/master/examples/java-demo) fo | Repository | Name | Version | |------------|------|---------| | https://charts.helm.sh/stable | redis | 10.5.6 | -| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.34.0 | -| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.34.0 | +| https://feast-helm-charts.storage.googleapis.com | feature-server(feature-server) | 0.35.0 | +| https://feast-helm-charts.storage.googleapis.com | transformation-service(transformation-service) | 0.35.0 | ## Values diff --git a/infra/charts/feast/charts/feature-server/Chart.yaml b/infra/charts/feast/charts/feature-server/Chart.yaml index bfb33b6140..122c80f9fd 100644 --- a/infra/charts/feast/charts/feature-server/Chart.yaml +++ b/infra/charts/feast/charts/feature-server/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Feast Feature Server: Online feature serving service for Feast" name: feature-server -version: 0.34.0 -appVersion: v0.34.0 +version: 0.35.0 +appVersion: v0.35.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/feature-server/README.md b/infra/charts/feast/charts/feature-server/README.md index f768a46cd5..514fcc5727 100644 --- a/infra/charts/feast/charts/feature-server/README.md +++ b/infra/charts/feast/charts/feature-server/README.md @@ -1,6 +1,6 @@ # feature-server -![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square) +![Version: 0.35.0](https://img.shields.io/badge/Version-0.35.0-informational?style=flat-square) ![AppVersion: v0.35.0](https://img.shields.io/badge/AppVersion-v0.35.0-informational?style=flat-square) Feast Feature Server: Online feature serving service for Feast @@ -17,7 +17,7 @@ Feast Feature Server: Online feature serving service for Feast | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-server-java"` | Docker image for Feature Server repository | -| image.tag | string | `"0.34.0"` | Image tag | +| image.tag | string | `"0.35.0"` | Image tag | | ingress.grpc.annotations | object | `{}` | Extra annotations for the ingress | | ingress.grpc.auth.enabled | bool | `false` | Flag to enable auth | | ingress.grpc.class | string | `"nginx"` | Which ingress controller to use | @@ -64,4 +64,4 @@ Feast Feature Server: Online feature serving service for Feast | transformationService.port | int | `6566` | | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) +Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0) diff --git a/infra/charts/feast/charts/feature-server/values.yaml b/infra/charts/feast/charts/feature-server/values.yaml index 24b8da1e39..a14cc2d28f 100644 --- a/infra/charts/feast/charts/feature-server/values.yaml +++ b/infra/charts/feast/charts/feature-server/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Feature Server repository repository: feastdev/feature-server-java # image.tag -- Image tag - tag: 0.34.0 + tag: 0.35.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/charts/transformation-service/Chart.yaml b/infra/charts/feast/charts/transformation-service/Chart.yaml index 5d8f157a48..852045f52b 100644 --- a/infra/charts/feast/charts/transformation-service/Chart.yaml +++ b/infra/charts/feast/charts/transformation-service/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 description: "Transformation service: to compute on-demand features" name: transformation-service -version: 0.34.0 -appVersion: v0.34.0 +version: 0.35.0 +appVersion: v0.35.0 keywords: - machine learning - big data diff --git a/infra/charts/feast/charts/transformation-service/README.md b/infra/charts/feast/charts/transformation-service/README.md index cf8c7eaae8..758620d56a 100644 --- a/infra/charts/feast/charts/transformation-service/README.md +++ b/infra/charts/feast/charts/transformation-service/README.md @@ -1,6 +1,6 @@ # transformation-service -![Version: 0.34.0](https://img.shields.io/badge/Version-0.34.0-informational?style=flat-square) ![AppVersion: v0.34.0](https://img.shields.io/badge/AppVersion-v0.34.0-informational?style=flat-square) +![Version: 0.35.0](https://img.shields.io/badge/Version-0.35.0-informational?style=flat-square) ![AppVersion: v0.35.0](https://img.shields.io/badge/AppVersion-v0.35.0-informational?style=flat-square) Transformation service: to compute on-demand features @@ -13,7 +13,7 @@ Transformation service: to compute on-demand features | envOverrides | object | `{}` | Extra environment variables to set | | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | | image.repository | string | `"feastdev/feature-transformation-server"` | Docker image for Transformation Server repository | -| image.tag | string | `"0.34.0"` | Image tag | +| image.tag | string | `"0.35.0"` | Image tag | | nodeSelector | object | `{}` | Node labels for pod assignment | | podLabels | object | `{}` | Labels to be added to Feast Serving pods | | replicaCount | int | `1` | Number of pods that will be created | @@ -25,4 +25,4 @@ Transformation service: to compute on-demand features | service.type | string | `"ClusterIP"` | Kubernetes service type | ---------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) +Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0) diff --git a/infra/charts/feast/charts/transformation-service/values.yaml b/infra/charts/feast/charts/transformation-service/values.yaml index 6af9e569ea..e45ef47288 100644 --- a/infra/charts/feast/charts/transformation-service/values.yaml +++ b/infra/charts/feast/charts/transformation-service/values.yaml @@ -5,7 +5,7 @@ image: # image.repository -- Docker image for Transformation Server repository repository: feastdev/feature-transformation-server # image.tag -- Image tag - tag: 0.34.0 + tag: 0.35.0 # image.pullPolicy -- Image pull policy pullPolicy: IfNotPresent diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index b3236b0322..ec098f2f7b 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,12 +1,12 @@ dependencies: - name: feature-server alias: feature-server - version: 0.34.0 + version: 0.35.0 condition: feature-server.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: transformation-service alias: transformation-service - version: 0.34.0 + version: 0.35.0 condition: transformation-service.enabled repository: https://feast-helm-charts.storage.googleapis.com - name: redis diff --git a/infra/templates/README.md.jinja2 b/infra/templates/README.md.jinja2 index 47779d4eb7..1cce08ecfa 100644 --- a/infra/templates/README.md.jinja2 +++ b/infra/templates/README.md.jinja2 @@ -25,7 +25,7 @@ Feast allows ML platform teams to: * **Avoid data leakage** by generating point-in-time correct feature sets so data scientists can focus on feature engineering rather than debugging error-prone dataset joining logic. This ensure that future feature values do not leak to models during training. * **Decouple ML from data infrastructure** by providing a single data access layer that abstracts feature storage from feature retrieval, ensuring models remain portable as you move from training models to serving models, from batch models to realtime models, and from one data infra system to another. -Please see our [documentation](https://docs.feast.dev/) for more information about the project, or sign up for an [email newsletter](https://feast.dev/). +Please see our [documentation](https://docs.feast.dev/) for more information about the project. ## 📐 Architecture ![](docs/assets/feast_marchitecture.png) @@ -149,7 +149,6 @@ Please refer to the official documentation at [Documentation](https://docs.feast * [Tutorials](https://docs.feast.dev/tutorials/tutorials-overview) * [Running Feast with Snowflake/GCP/AWS](https://docs.feast.dev/how-to-guides/feast-snowflake-gcp-aws) * [Change Log](https://github.com/feast-dev/feast/blob/master/CHANGELOG.md) - * [Slack (#Feast)](https://slack.feast.dev/) ## 👋 Contributing Feast is a community project and is still under active development. Please have a look at our contributing and development guides if you want to contribute to the project: diff --git a/java/pom.xml b/java/pom.xml index 245d2d7cc6..59c6733784 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -35,7 +35,7 @@ - 0.34.0 + 0.35.0 https://github.com/feast-dev/feast UTF-8 @@ -61,7 +61,7 @@ 1.5.24 3.14.7 3.10 - 2.12.6 + 2.14.0 2.3.1 1.3.2 2.0.1.Final diff --git a/java/serving/pom.xml b/java/serving/pom.xml index 79f942d491..19e54e1362 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -274,7 +274,7 @@ org.apache.avro avro - 1.10.2 + 1.11.3 diff --git a/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java b/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java index 564159ceed..8beb91e485 100644 --- a/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java +++ b/java/serving/src/main/java/feast/serving/service/config/ServingServiceV2Module.java @@ -54,7 +54,6 @@ public ServingServiceV2 registryBasedServingServiceV2( break; case REDIS: RedisClientAdapter redisClient = RedisClient.create(store.getRedisConfig()); - log.info("Created EntityKeySerializerV2"); retriever = new RedisOnlineRetriever( applicationProperties.getFeast().getProject(), @@ -69,8 +68,6 @@ public ServingServiceV2 registryBasedServingServiceV2( store.getType())); } - log.info("Working Directory = " + System.getProperty("user.dir")); - final OnlineTransformationService onlineTransformationService = new OnlineTransformationService( applicationProperties.getFeast().getTransformationServiceEndpoint(), diff --git a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt index 94e4771de2..6ba2c53d81 100644 --- a/java/serving/src/test/resources/docker-compose/feast10/requirements.txt +++ b/java/serving/src/test/resources/docker-compose/feast10/requirements.txt @@ -1,5 +1,5 @@ # for source generation -pyarrow==6.0.0 +pyarrow==14.0.1 # temp fixes proto-plus diff --git a/protos/feast/serving/GrpcServer.proto b/protos/feast/serving/GrpcServer.proto index cd0274c5c7..34edb4ebe9 100644 --- a/protos/feast/serving/GrpcServer.proto +++ b/protos/feast/serving/GrpcServer.proto @@ -1,5 +1,7 @@ syntax = "proto3"; +import "feast/serving/ServingService.proto"; + message PushRequest { map features = 1; string stream_feature_view = 2; @@ -8,7 +10,7 @@ message PushRequest { } message PushResponse { - bool status = 1; + bool status = 1; } message WriteToOnlineStoreRequest { @@ -18,10 +20,11 @@ message WriteToOnlineStoreRequest { } message WriteToOnlineStoreResponse { - bool status = 1; + bool status = 1; } service GrpcFeatureServer { - rpc Push (PushRequest) returns (PushResponse) {}; - rpc WriteToOnlineStore (WriteToOnlineStoreRequest) returns (WriteToOnlineStoreResponse); + rpc Push (PushRequest) returns (PushResponse) {}; + rpc WriteToOnlineStore (WriteToOnlineStoreRequest) returns (WriteToOnlineStoreResponse); + rpc GetOnlineFeatures (feast.serving.GetOnlineFeaturesRequest) returns (feast.serving.GetOnlineFeaturesResponse); } \ No newline at end of file diff --git a/protos/feast/serving/ServingService.proto b/protos/feast/serving/ServingService.proto index 0eef3cd883..154d850099 100644 --- a/protos/feast/serving/ServingService.proto +++ b/protos/feast/serving/ServingService.proto @@ -105,6 +105,8 @@ message GetOnlineFeaturesResponse { repeated FieldStatus statuses = 2; repeated google.protobuf.Timestamp event_timestamps = 3; } + + bool status = 3; } message GetOnlineFeaturesResponseMetadata { diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 8c2c326b59..2eb2c27bcb 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -18,10 +18,10 @@ from typing import List, Optional import click -import pkg_resources import yaml from colorama import Fore, Style from dateutil import parser +from importlib_metadata import version as importlib_version from pygments import formatters, highlight, lexers from feast import utils @@ -68,7 +68,7 @@ def format_options(self, ctx: click.Context, formatter: click.HelpFormatter): ) @click.option( "--log-level", - default="info", + default="warning", help="The logging level. One of DEBUG, INFO, WARNING, ERROR, and CRITICAL (case-insensitive).", ) @click.option( @@ -86,8 +86,6 @@ def cli( Feast CLI For more information, see our public docs at https://docs.feast.dev/ - - For any questions, you can reach us at https://slack.feast.dev/ """ ctx.ensure_object(dict) ctx.obj["CHDIR"] = Path.cwd() if chdir is None else Path(chdir).absolute() @@ -122,7 +120,7 @@ def version(): """ Display Feast SDK version """ - print(f'Feast SDK Version: "{pkg_resources.get_distribution("feast")}"') + print(f'Feast SDK Version: "{importlib_version("feast")}"') @cli.command() @@ -665,6 +663,14 @@ def init_command(project_directory, minimal: bool, template: str): show_default=True, help="Timeout for keep alive", ) +@click.option( + "--registry_ttl_sec", + "-r", + help="Number of seconds after which the registry is refreshed", + type=click.INT, + default=5, + show_default=True, +) @click.pass_context def serve_command( ctx: click.Context, @@ -675,6 +681,7 @@ def serve_command( no_feature_log: bool, workers: int, keep_alive_timeout: int, + registry_ttl_sec: int = 5, ): """Start a feature server locally on a given port.""" store = create_feature_store(ctx) @@ -687,6 +694,7 @@ def serve_command( no_feature_log=no_feature_log, workers=workers, keep_alive_timeout=keep_alive_timeout, + registry_ttl_sec=registry_ttl_sec, ) @@ -707,15 +715,24 @@ def serve_command( show_default=False, help="The maximum number of threads that can be used to execute the gRPC calls", ) +@click.option( + "--registry_ttl_sec", + "-r", + help="Number of seconds after which the registry is refreshed", + type=click.INT, + default=5, + show_default=True, +) @click.pass_context def listen_command( ctx: click.Context, address: str, max_workers: int, + registry_ttl_sec: int, ): """Start a gRPC feature server to ingest streaming features on given address""" store = create_feature_store(ctx) - server = get_grpc_server(address, store, max_workers) + server = get_grpc_server(address, store, max_workers, registry_ttl_sec) server.start() server.wait_for_termination() diff --git a/sdk/python/feast/feature_server.py b/sdk/python/feast/feature_server.py index 3abca1d6e8..618aefb2f2 100644 --- a/sdk/python/feast/feature_server.py +++ b/sdk/python/feast/feature_server.py @@ -1,9 +1,12 @@ import json +import threading import traceback import warnings +from typing import List, Optional import gunicorn.app.base import pandas as pd +from dateutil import parser from fastapi import FastAPI, HTTPException, Request, Response, status from fastapi.logger import logger from fastapi.params import Depends @@ -11,7 +14,7 @@ from pydantic import BaseModel import feast -from feast import proto_json +from feast import proto_json, utils from feast.data_source import PushMode from feast.errors import PushSourceNotFoundException from feast.protos.feast.serving.ServingService_pb2 import GetOnlineFeaturesRequest @@ -31,14 +34,48 @@ class PushFeaturesRequest(BaseModel): to: str = "online" -def get_app(store: "feast.FeatureStore"): +class MaterializeRequest(BaseModel): + start_ts: str + end_ts: str + feature_views: Optional[List[str]] = None + + +class MaterializeIncrementalRequest(BaseModel): + end_ts: str + feature_views: Optional[List[str]] = None + + +def get_app(store: "feast.FeatureStore", registry_ttl_sec: int = 5): proto_json.patch() app = FastAPI() + # Asynchronously refresh registry, notifying shutdown and canceling the active timer if the app is shutting down + registry_proto = None + shutting_down = False + active_timer: Optional[threading.Timer] = None async def get_body(request: Request): return await request.body() + def async_refresh(): + store.refresh_registry() + nonlocal registry_proto + registry_proto = store.registry.proto() + if shutting_down: + return + nonlocal active_timer + active_timer = threading.Timer(registry_ttl_sec, async_refresh) + active_timer.start() + + @app.on_event("shutdown") + def shutdown_event(): + nonlocal shutting_down + shutting_down = True + if active_timer: + active_timer.cancel() + + async_refresh() + @app.post("/get-online-features") def get_online_features(body=Depends(get_body)): try: @@ -134,12 +171,43 @@ def write_to_online_store(body=Depends(get_body)): def health(): return Response(status_code=status.HTTP_200_OK) + @app.post("/materialize") + def materialize(body=Depends(get_body)): + try: + request = MaterializeRequest(**json.loads(body)) + store.materialize( + utils.make_tzaware(parser.parse(request.start_ts)), + utils.make_tzaware(parser.parse(request.end_ts)), + request.feature_views, + ) + except Exception as e: + # Print the original exception on the server side + logger.exception(traceback.format_exc()) + # Raise HTTPException to return the error message to the client + raise HTTPException(status_code=500, detail=str(e)) + + @app.post("/materialize-incremental") + def materialize_incremental(body=Depends(get_body)): + try: + request = MaterializeIncrementalRequest(**json.loads(body)) + store.materialize_incremental( + utils.make_tzaware(parser.parse(request.end_ts)), request.feature_views + ) + except Exception as e: + # Print the original exception on the server side + logger.exception(traceback.format_exc()) + # Raise HTTPException to return the error message to the client + raise HTTPException(status_code=500, detail=str(e)) + return app class FeastServeApplication(gunicorn.app.base.BaseApplication): def __init__(self, store: "feast.FeatureStore", **options): - self._app = get_app(store=store) + self._app = get_app( + store=store, + registry_ttl_sec=options.get("registry_ttl_sec", 5), + ) self._options = options super().__init__() @@ -161,6 +229,7 @@ def start_server( no_access_log: bool, workers: int, keep_alive_timeout: int, + registry_ttl_sec: int = 5, ): FeastServeApplication( store=store, @@ -168,4 +237,5 @@ def start_server( accesslog=None if no_access_log else "-", workers=workers, keepalive=keep_alive_timeout, + registry_ttl_sec=registry_ttl_sec, ).run() diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 70f7d3dcb7..d3f98f8032 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -287,7 +287,11 @@ def _list_feature_views( for fv in self._registry.list_feature_views( self.project, allow_cache=allow_cache ): - if hide_dummy_entity and fv.entities[0] == DUMMY_ENTITY_NAME: + if ( + hide_dummy_entity + and fv.entities + and fv.entities[0] == DUMMY_ENTITY_NAME + ): fv.entities = [] fv.entity_columns = [] feature_views.append(fv) @@ -2224,6 +2228,7 @@ def serve( no_feature_log: bool, workers: int, keep_alive_timeout: int, + registry_ttl_sec: int, ) -> None: """Start the feature consumption server locally on a given port.""" type_ = type_.lower() @@ -2239,6 +2244,7 @@ def serve( no_access_log=no_access_log, workers=workers, keep_alive_timeout=keep_alive_timeout, + registry_ttl_sec=registry_ttl_sec, ) @log_exceptions_and_usage diff --git a/sdk/python/feast/infra/contrib/grpc_server.py b/sdk/python/feast/infra/contrib/grpc_server.py index 2017f1095b..27ac45e77c 100644 --- a/sdk/python/feast/infra/contrib/grpc_server.py +++ b/sdk/python/feast/infra/contrib/grpc_server.py @@ -1,12 +1,14 @@ import logging +import threading from concurrent import futures +from typing import Optional import grpc import pandas as pd from grpc_health.v1 import health, health_pb2_grpc from feast.data_source import PushMode -from feast.errors import PushSourceNotFoundException +from feast.errors import FeatureServiceNotFoundException, PushSourceNotFoundException from feast.feature_store import FeatureStore from feast.protos.feast.serving.GrpcServer_pb2 import ( PushResponse, @@ -16,6 +18,12 @@ GrpcFeatureServerServicer, add_GrpcFeatureServerServicer_to_server, ) +from feast.protos.feast.serving.ServingService_pb2 import ( + GetOnlineFeaturesRequest, + GetOnlineFeaturesResponse, +) + +logger = logging.getLogger(__name__) def parse(features): @@ -28,10 +36,16 @@ def parse(features): class GrpcFeatureServer(GrpcFeatureServerServicer): fs: FeatureStore - def __init__(self, fs: FeatureStore): + _shuting_down: bool = False + _active_timer: Optional[threading.Timer] = None + + def __init__(self, fs: FeatureStore, registry_ttl_sec: int = 5): self.fs = fs + self.registry_ttl_sec = registry_ttl_sec super().__init__() + self._async_refresh() + def Push(self, request, context): try: df = parse(request.features) @@ -53,19 +67,19 @@ def Push(self, request, context): to=to, ) except PushSourceNotFoundException as e: - logging.exception(str(e)) + logger.exception(str(e)) context.set_code(grpc.StatusCode.INVALID_ARGUMENT) context.set_details(str(e)) return PushResponse(status=False) except Exception as e: - logging.exception(str(e)) + logger.exception(str(e)) context.set_code(grpc.StatusCode.INTERNAL) context.set_details(str(e)) return PushResponse(status=False) return PushResponse(status=True) def WriteToOnlineStore(self, request, context): - logging.warning( + logger.warning( "write_to_online_store is deprecated. Please consider using Push instead" ) try: @@ -76,16 +90,55 @@ def WriteToOnlineStore(self, request, context): allow_registry_cache=request.allow_registry_cache, ) except Exception as e: - logging.exception(str(e)) + logger.exception(str(e)) context.set_code(grpc.StatusCode.INTERNAL) context.set_details(str(e)) return PushResponse(status=False) return WriteToOnlineStoreResponse(status=True) + def GetOnlineFeatures(self, request: GetOnlineFeaturesRequest, context): + if request.HasField("feature_service"): + logger.info(f"Requesting feature service: {request.feature_service}") + try: + features = self.fs.get_feature_service( + request.feature_service, allow_cache=True + ) + except FeatureServiceNotFoundException as e: + logger.error(f"Feature service {request.feature_service} not found") + context.set_code(grpc.StatusCode.INTERNAL) + context.set_details(str(e)) + return GetOnlineFeaturesResponse() + else: + features = list(request.features.val) + + result = self.fs._get_online_features( + features, + request.entities, + request.full_feature_names, + ).proto + + return result + + def _async_refresh(self): + self.fs.refresh_registry() + if self._shuting_down: + return + self._active_timer = threading.Timer(self.registry_ttl_sec, self._async_refresh) + self._active_timer.start() -def get_grpc_server(address: str, fs: FeatureStore, max_workers: int): + +def get_grpc_server( + address: str, + fs: FeatureStore, + max_workers: int, + registry_ttl_sec: int, +): + logger.info(f"Initializing gRPC server on {address}") server = grpc.server(futures.ThreadPoolExecutor(max_workers=max_workers)) - add_GrpcFeatureServerServicer_to_server(GrpcFeatureServer(fs), server) + add_GrpcFeatureServerServicer_to_server( + GrpcFeatureServer(fs, registry_ttl_sec=registry_ttl_sec), + server, + ) health_servicer = health.HealthServicer( experimental_non_blocking=True, experimental_thread_pool=futures.ThreadPoolExecutor(max_workers=max_workers), diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile index 963924f38d..a26661ead3 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile @@ -25,5 +25,5 @@ COPY README.md README.md # git dir to infer the version of feast we're installing. # https://github.com/pypa/setuptools_scm#usage-from-docker # I think it also assumes that this dockerfile is being built from the root of the directory. -RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir -e '.[aws,gcp,bytewax]' +RUN --mount=source=.git,target=.git,type=bind pip3 install --no-cache-dir '.[aws,gcp,bytewax,snowflake]' diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py index bf5229303a..6fc53b67f2 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py @@ -1,17 +1,20 @@ +import logging +import os from typing import List import pyarrow as pa import pyarrow.parquet as pq -import s3fs from bytewax.dataflow import Dataflow # type: ignore from bytewax.execution import cluster_main -from bytewax.inputs import ManualInputConfig, distribute +from bytewax.inputs import ManualInputConfig from bytewax.outputs import ManualOutputConfig -from tqdm import tqdm from feast import FeatureStore, FeatureView, RepoConfig from feast.utils import _convert_arrow_to_proto, _run_pyarrow_field_mapping +logger = logging.getLogger(__name__) +DEFAULT_BATCH_SIZE = 1000 + class BytewaxMaterializationDataflow: def __init__( @@ -19,57 +22,56 @@ def __init__( config: RepoConfig, feature_view: FeatureView, paths: List[str], + worker_index: int, ): self.config = config self.feature_store = FeatureStore(config=config) self.feature_view = feature_view + self.worker_index = worker_index self.paths = paths + self.mini_batch_size = int( + os.getenv("BYTEWAX_MINI_BATCH_SIZE", DEFAULT_BATCH_SIZE) + ) self._run_dataflow() def process_path(self, path): - fs = s3fs.S3FileSystem() - dataset = pq.ParquetDataset(path, filesystem=fs, use_legacy_dataset=False) + logger.info(f"Processing path {path}") + dataset = pq.ParquetDataset(path, use_legacy_dataset=False) batches = [] for fragment in dataset.fragments: - for batch in fragment.to_table().to_batches(): + for batch in fragment.to_table().to_batches( + max_chunksize=self.mini_batch_size + ): batches.append(batch) return batches def input_builder(self, worker_index, worker_count, _state): - worker_paths = distribute(self.paths, worker_index, worker_count) - for path in worker_paths: - yield None, path - - return + return [(None, self.paths[self.worker_index])] def output_builder(self, worker_index, worker_count): - def output_fn(batch): - table = pa.Table.from_batches([batch]) + def output_fn(mini_batch): + table: pa.Table = pa.Table.from_batches([mini_batch]) if self.feature_view.batch_source.field_mapping is not None: table = _run_pyarrow_field_mapping( table, self.feature_view.batch_source.field_mapping ) - join_key_to_value_type = { entity.name: entity.dtype.to_value_type() for entity in self.feature_view.entity_columns } - rows_to_write = _convert_arrow_to_proto( table, self.feature_view, join_key_to_value_type ) - provider = self.feature_store._get_provider() - with tqdm(total=len(rows_to_write)) as progress: - provider.online_write_batch( - config=self.config, - table=self.feature_view, - data=rows_to_write, - progress=progress.update, - ) + self.feature_store._get_provider().online_write_batch( + config=self.config, + table=self.feature_view, + data=rows_to_write, + progress=None, + ) return output_fn diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py index b222128bbb..5c7a719532 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_engine.py @@ -1,11 +1,14 @@ +import logging import uuid from datetime import datetime +from time import sleep from typing import Callable, List, Literal, Sequence, Union import yaml from kubernetes import client from kubernetes import config as k8s_config from kubernetes import utils +from kubernetes.client.exceptions import ApiException from kubernetes.utils import FailToCreateError from pydantic import StrictStr from tqdm import tqdm @@ -16,6 +19,7 @@ from feast.infra.materialization.batch_materialization_engine import ( BatchMaterializationEngine, MaterializationJob, + MaterializationJobStatus, MaterializationTask, ) from feast.infra.offline_stores.offline_store import OfflineStore @@ -23,10 +27,12 @@ from feast.infra.registry.base_registry import BaseRegistry from feast.repo_config import FeastConfigBaseModel from feast.stream_feature_view import StreamFeatureView -from feast.utils import _get_column_names, get_default_yaml_file_path +from feast.utils import _get_column_names from .bytewax_materialization_job import BytewaxMaterializationJob +logger = logging.getLogger(__name__) + class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): """Batch Materialization Engine config for Bytewax""" @@ -61,6 +67,30 @@ class BytewaxMaterializationEngineConfig(FeastConfigBaseModel): include_security_context_capabilities: bool = True """ (optional) Include security context capabilities in the init and job container spec """ + labels: dict = {} + """ (optional) additional labels to append to kubernetes objects """ + + max_parallelism: int = 10 + """ (optional) Maximum number of pods allowed to run in parallel""" + + synchronous: bool = False + """ (optional) If true, wait for materialization for one feature to complete before moving to the next """ + + retry_limit: int = 2 + """ (optional) Maximum number of times to retry a materialization worker pod""" + + mini_batch_size: int = 1000 + """ (optional) Number of rows to process per write operation (default 1000)""" + + active_deadline_seconds: int = 86400 + """ (optional) Maximum amount of time a materialization job is allowed to run""" + + job_batch_size: int = 100 + """ (optional) Maximum number of pods to process per job. Only applies to synchronous materialization""" + + print_pod_logs_on_failure: bool = True + """(optional) Print pod logs on job failure. Only applies to synchronous materialization""" + class BytewaxMaterializationEngine(BatchMaterializationEngine): def __init__( @@ -81,8 +111,7 @@ def __init__( self.offline_store = offline_store self.online_store = online_store - # TODO: Configure k8s here - k8s_config.load_kube_config() + k8s_config.load_config() self.k8s_client = client.api_client.ApiClient() self.v1 = client.CoreV1Api(self.k8s_client) @@ -164,8 +193,98 @@ def _materialize_one( ) paths = offline_job.to_remote_storage() + if self.batch_engine_config.synchronous: + offset = 0 + total_pods = len(paths) + batch_size = self.batch_engine_config.job_batch_size + if batch_size < 1: + raise ValueError("job_batch_size must be a value greater than 0") + if batch_size < self.batch_engine_config.max_parallelism: + logger.warning( + "job_batch_size is less than max_parallelism. Setting job_batch_size = max_parallelism" + ) + batch_size = self.batch_engine_config.max_parallelism + + while True: + next_offset = min(offset + batch_size, total_pods) + job = self._await_path_materialization( + paths[offset:next_offset], + feature_view, + offset, + next_offset, + total_pods, + ) + offset += batch_size + if ( + offset >= total_pods + or job.status() == MaterializationJobStatus.ERROR + ): + break + else: + job_id = str(uuid.uuid4()) + job = self._create_kubernetes_job(job_id, paths, feature_view) + + return job + + def _await_path_materialization( + self, paths, feature_view, batch_start, batch_end, total_pods + ): job_id = str(uuid.uuid4()) - return self._create_kubernetes_job(job_id, paths, feature_view) + job = self._create_kubernetes_job(job_id, paths, feature_view) + + try: + while job.status() in ( + MaterializationJobStatus.WAITING, + MaterializationJobStatus.RUNNING, + ): + logger.info( + f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " + f"(of {total_pods}) running..." + ) + sleep(30) + logger.info( + f"{feature_view.name} materialization for pods {batch_start}-{batch_end} " + f"(of {total_pods}) complete with status {job.status()}" + ) + except BaseException as e: + logger.info(f"Deleting job {job.job_id()}") + try: + self.batch_v1.delete_namespaced_job(job.job_id(), self.namespace) + except ApiException as ae: + logger.warning(f"Could not delete job due to API Error: {ae.body}") + raise e + finally: + logger.info(f"Deleting configmap {self._configmap_name(job_id)}") + try: + self.v1.delete_namespaced_config_map( + self._configmap_name(job_id), self.namespace + ) + except ApiException as ae: + logger.warning( + f"Could not delete configmap due to API Error: {ae.body}" + ) + + if ( + job.status() == MaterializationJobStatus.ERROR + and self.batch_engine_config.print_pod_logs_on_failure + ): + self._print_pod_logs(job.job_id(), feature_view, batch_start) + + return job + + def _print_pod_logs(self, job_id, feature_view, offset=0): + pods_list = self.v1.list_namespaced_pod( + namespace=self.namespace, + label_selector=f"job-name={job_id}", + ).items + for i, pod in enumerate(pods_list): + logger.info(f"Logging output for {feature_view.name} pod {offset+i}") + try: + logger.info( + self.v1.read_namespaced_pod_log(pod.metadata.name, self.namespace) + ) + except ApiException as e: + logger.warning(f"Could not retrieve pod logs due to: {e.body}") def _create_kubernetes_job(self, job_id, paths, feature_view): try: @@ -179,6 +298,9 @@ def _create_kubernetes_job(self, job_id, paths, feature_view): len(paths), # Create a pod for each parquet file self.batch_engine_config.env, ) + logger.info( + f"Created job `dataflow-{job_id}` on namespace `{self.namespace}`" + ) except FailToCreateError as failures: return BytewaxMaterializationJob(job_id, self.namespace, error=failures) @@ -187,23 +309,19 @@ def _create_kubernetes_job(self, job_id, paths, feature_view): def _create_configuration_map(self, job_id, paths, feature_view, namespace): """Create a Kubernetes configmap for this job""" - repo_path = self.repo_config.repo_path - assert repo_path - feature_store_path = get_default_yaml_file_path(repo_path) - feature_store_configuration = feature_store_path.read_text() + feature_store_configuration = yaml.dump(self.repo_config.dict()) materialization_config = yaml.dump( {"paths": paths, "feature_view": feature_view.name} ) + labels = {"feast-bytewax-materializer": "configmap"} configmap_manifest = { "kind": "ConfigMap", "apiVersion": "v1", "metadata": { - "name": f"feast-{job_id}", - "labels": { - "feast-bytewax-materializer": "configmap", - }, + "name": self._configmap_name(job_id), + "labels": {**labels, **self.batch_engine_config.labels}, }, "data": { "feature_store.yaml": feature_store_configuration, @@ -215,7 +333,10 @@ def _create_configuration_map(self, job_id, paths, feature_view, namespace): body=configmap_manifest, ) - def _create_job_definition(self, job_id, namespace, pods, env): + def _configmap_name(self, job_id): + return f"feast-{job_id}" + + def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0): """Create a kubernetes job definition.""" job_env = [ {"name": "RUST_BACKTRACE", "value": "full"}, @@ -239,7 +360,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): }, { "name": "BYTEWAX_REPLICAS", - "value": f"{pods}", + "value": "1", }, { "name": "BYTEWAX_KEEP_CONTAINER_ALIVE", @@ -249,6 +370,10 @@ def _create_job_definition(self, job_id, namespace, pods, env): "name": "BYTEWAX_STATEFULSET_NAME", "value": f"dataflow-{job_id}", }, + { + "name": "BYTEWAX_MINI_BATCH_SIZE", + "value": str(self.batch_engine_config.mini_batch_size), + }, ] # Add any Feast configured environment variables job_env.extend(env) @@ -260,27 +385,27 @@ def _create_job_definition(self, job_id, namespace, pods, env): "drop": ["ALL"], } + job_labels = {"feast-bytewax-materializer": "job"} + pod_labels = {"feast-bytewax-materializer": "pod"} job_definition = { "apiVersion": "batch/v1", "kind": "Job", "metadata": { "name": f"dataflow-{job_id}", "namespace": namespace, - "labels": { - "feast-bytewax-materializer": "job", - }, + "labels": {**job_labels, **self.batch_engine_config.labels}, }, "spec": { "ttlSecondsAfterFinished": 3600, + "backoffLimit": self.batch_engine_config.retry_limit, "completions": pods, - "parallelism": pods, + "parallelism": min(pods, self.batch_engine_config.max_parallelism), + "activeDeadlineSeconds": self.batch_engine_config.active_deadline_seconds, "completionMode": "Indexed", "template": { "metadata": { "annotations": self.batch_engine_config.annotations, - "labels": { - "feast-bytewax-materializer": "pod", - }, + "labels": {**pod_labels, **self.batch_engine_config.labels}, }, "spec": { "restartPolicy": "Never", @@ -314,7 +439,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): }, { "mountPath": "/var/feast/", - "name": f"feast-{job_id}", + "name": self._configmap_name(job_id), }, ], } @@ -345,7 +470,7 @@ def _create_job_definition(self, job_id, namespace, pods, env): {"mountPath": "/etc/bytewax", "name": "hostfile"}, { "mountPath": "/var/feast/", - "name": f"feast-{job_id}", + "name": self._configmap_name(job_id), }, ], } @@ -355,13 +480,13 @@ def _create_job_definition(self, job_id, namespace, pods, env): { "configMap": { "defaultMode": 420, - "name": f"feast-{job_id}", + "name": self._configmap_name(job_id), }, "name": "python-files", }, { - "configMap": {"name": f"feast-{job_id}"}, - "name": f"feast-{job_id}", + "configMap": {"name": self._configmap_name(job_id)}, + "name": self._configmap_name(job_id), }, ], }, diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py index 77d2149eb5..da969d5a88 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_job.py @@ -35,12 +35,24 @@ def status(self): if job_status.active is not None: if job_status.completion_time is None: return MaterializationJobStatus.RUNNING - elif job_status.failed is not None: - return MaterializationJobStatus.ERROR - elif job_status.active is None and job_status.succeeded is not None: - if job_status.conditions[0].type == "Complete": + else: + if ( + job_status.completion_time is not None + and job_status.conditions[0].type == "Complete" + ): return MaterializationJobStatus.SUCCEEDED + if ( + job_status.conditions is not None + and job_status.conditions[0].type == "Failed" + ): + self._error = Exception( + f"Job {self.job_id()} failed with reason: " + f"{job_status.conditions[0].message}" + ) + return MaterializationJobStatus.ERROR + return MaterializationJobStatus.WAITING + def should_be_retried(self): return False diff --git a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py index e3d95e2a75..9d9b328c0e 100644 --- a/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py +++ b/sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py @@ -1,3 +1,6 @@ +import logging +import os + import yaml from feast import FeatureStore, RepoConfig @@ -6,6 +9,8 @@ ) if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + with open("/var/feast/feature_store.yaml") as f: feast_config = yaml.safe_load(f) @@ -19,4 +24,5 @@ config, store.get_feature_view(bytewax_config["feature_view"]), bytewax_config["paths"], + int(os.environ["JOB_COMPLETION_INDEX"]), ) diff --git a/sdk/python/feast/infra/offline_stores/bigquery.py b/sdk/python/feast/infra/offline_stores/bigquery.py index 5913b60f62..b7910c391c 100644 --- a/sdk/python/feast/infra/offline_stores/bigquery.py +++ b/sdk/python/feast/infra/offline_stores/bigquery.py @@ -19,7 +19,7 @@ import pandas as pd import pyarrow import pyarrow.parquet -from pydantic import StrictStr, validator +from pydantic import ConstrainedStr, StrictStr, validator from pydantic.typing import Literal from tenacity import Retrying, retry_if_exception_type, stop_after_delay, wait_fixed @@ -72,6 +72,13 @@ def get_http_client_info(): return http_client_info.ClientInfo(user_agent=get_user_agent()) +class BigQueryTableCreateDisposition(ConstrainedStr): + """Custom constraint for table_create_disposition. To understand more, see: + https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad.FIELDS.create_disposition""" + + values = {"CREATE_NEVER", "CREATE_IF_NEEDED"} + + class BigQueryOfflineStoreConfig(FeastConfigBaseModel): """Offline store config for GCP BigQuery""" @@ -95,6 +102,9 @@ class BigQueryOfflineStoreConfig(FeastConfigBaseModel): gcs_staging_location: Optional[str] = None """ (optional) GCS location used for offloading BigQuery results as parquet files.""" + table_create_disposition: Optional[BigQueryTableCreateDisposition] = None + """ (optional) Specifies whether the job is allowed to create new tables. The default value is CREATE_IF_NEEDED.""" + @validator("billing_project_id") def project_id_exists(cls, v, values, **kwargs): if v and not values["project_id"]: @@ -324,6 +334,7 @@ def write_logged_features( job_config = bigquery.LoadJobConfig( source_format=bigquery.SourceFormat.PARQUET, schema=arrow_schema_to_bq_schema(source.get_schema(registry)), + create_disposition=config.offline_store.table_create_disposition, time_partitioning=bigquery.TimePartitioning( type_=bigquery.TimePartitioningType.DAY, field=source.get_log_timestamp_column(), @@ -342,7 +353,11 @@ def write_logged_features( return with tempfile.TemporaryFile() as parquet_temp_file: - pyarrow.parquet.write_table(table=data, where=parquet_temp_file) + # In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4. + # Set the coerce_timestamps to "us"(microseconds) for backward compatibility. + pyarrow.parquet.write_table( + table=data, where=parquet_temp_file, coerce_timestamps="us" + ) parquet_temp_file.seek(0) @@ -384,11 +399,16 @@ def offline_write_batch( job_config = bigquery.LoadJobConfig( source_format=bigquery.SourceFormat.PARQUET, schema=arrow_schema_to_bq_schema(pa_schema), + create_disposition=config.offline_store.table_create_disposition, write_disposition="WRITE_APPEND", # Default but included for clarity ) with tempfile.TemporaryFile() as parquet_temp_file: - pyarrow.parquet.write_table(table=table, where=parquet_temp_file) + # In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4. + # Set the coerce_timestamps to "us"(microseconds) for backward compatibility. + pyarrow.parquet.write_table( + table=table, where=parquet_temp_file, coerce_timestamps="us" + ) parquet_temp_file.seek(0) @@ -503,7 +523,7 @@ def to_bigquery( temp_dest_table = f"{tmp_dest['projectId']}.{tmp_dest['datasetId']}.{tmp_dest['tableId']}" # persist temp table - sql = f"CREATE TABLE `{dest}` AS SELECT * FROM {temp_dest_table}" + sql = f"CREATE TABLE `{dest}` AS SELECT * FROM `{temp_dest_table}`" self._execute_query(sql, timeout=timeout) print(f"Done writing to '{dest}'.") diff --git a/sdk/python/feast/infra/offline_stores/file_source.py b/sdk/python/feast/infra/offline_stores/file_source.py index d8522fb445..ac824b359f 100644 --- a/sdk/python/feast/infra/offline_stores/file_source.py +++ b/sdk/python/feast/infra/offline_stores/file_source.py @@ -158,7 +158,7 @@ def get_table_column_names_and_types( # Adding support for different file format path # based on S3 filesystem if filesystem is None: - schema = ParquetDataset(path).schema + schema = ParquetDataset(path, use_legacy_dataset=False).schema if hasattr(schema, "names") and hasattr(schema, "types"): # Newer versions of pyarrow doesn't have this method, # but this field is good enough. diff --git a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py index aff0c6c42c..1da9de89a8 100644 --- a/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py +++ b/sdk/python/feast/infra/online_stores/contrib/hbase_online_store/hbase.py @@ -3,14 +3,16 @@ from datetime import datetime from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple -from happybase import Connection +from happybase import ConnectionPool +from happybase.connection import DEFAULT_PROTOCOL, DEFAULT_TRANSPORT +from pydantic import StrictStr from pydantic.typing import Literal from feast import Entity from feast.feature_view import FeatureView -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.online_stores.helpers import compute_entity_id from feast.infra.online_stores.online_store import OnlineStore -from feast.infra.utils.hbase_utils import HbaseConstants, HbaseUtils +from feast.infra.utils.hbase_utils import HBaseConnector, HbaseConstants from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import FeastConfigBaseModel, RepoConfig @@ -23,35 +25,20 @@ class HbaseOnlineStoreConfig(FeastConfigBaseModel): type: Literal["hbase"] = "hbase" """Online store type selector""" - host: str + host: StrictStr """Hostname of Hbase Thrift server""" - port: str + port: StrictStr """Port in which Hbase Thrift server is running""" + connection_pool_size: int = 4 + """Number of connections to Hbase Thrift server to keep in the connection pool""" -class HbaseConnection: - """ - Hbase connecttion to connect to hbase. - - Attributes: - store_config: Online store config for Hbase store. - """ - - def __init__(self, store_config: HbaseOnlineStoreConfig): - self._store_config = store_config - self._real_conn = Connection( - host=store_config.host, port=int(store_config.port) - ) + protocol: StrictStr = DEFAULT_PROTOCOL + """Protocol used to communicate with Hbase Thrift server""" - @property - def real_conn(self) -> Connection: - """Stores the real happybase Connection to connect to hbase.""" - return self._real_conn - - def close(self) -> None: - """Close the happybase connection.""" - self.real_conn.close() + transport: StrictStr = DEFAULT_TRANSPORT + """Transport used to communicate with Hbase Thrift server""" class HbaseOnlineStore(OnlineStore): @@ -62,7 +49,7 @@ class HbaseOnlineStore(OnlineStore): _conn: Happybase Connection to connect to hbase thrift server. """ - _conn: Connection = None + _conn: ConnectionPool = None def _get_conn(self, config: RepoConfig): """ @@ -76,7 +63,13 @@ def _get_conn(self, config: RepoConfig): assert isinstance(store_config, HbaseOnlineStoreConfig) if not self._conn: - self._conn = Connection(host=store_config.host, port=int(store_config.port)) + self._conn = ConnectionPool( + host=store_config.host, + port=int(store_config.port), + size=int(store_config.connection_pool_size), + protocol=store_config.protocol, + transport=store_config.transport, + ) return self._conn @log_exceptions_and_usage(online_store="hbase") @@ -102,16 +95,17 @@ def online_write_batch( the online store. Can be used to display progress. """ - hbase = HbaseUtils(self._get_conn(config)) + hbase = HBaseConnector(self._get_conn(config)) project = config.project - table_name = _table_id(project, table) + table_name = self._table_id(project, table) b = hbase.batch(table_name) for entity_key, values, timestamp, created_ts in data: - row_key = serialize_entity_key( + row_key = self._hbase_row_key( entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ).hex() + feature_view_name=table.name, + config=config, + ) values_dict = {} for feature_name, val in values.items(): values_dict[ @@ -133,6 +127,9 @@ def online_write_batch( b.put(row_key, values_dict) b.send() + if progress: + progress(len(data)) + @log_exceptions_and_usage(online_store="hbase") def online_read( self, @@ -150,17 +147,18 @@ def online_read( entity_keys: a list of entity keys that should be read from the FeatureStore. requested_features: a list of requested feature names. """ - hbase = HbaseUtils(self._get_conn(config)) + hbase = HBaseConnector(self._get_conn(config)) project = config.project - table_name = _table_id(project, table) + table_name = self._table_id(project, table) result: List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]] = [] row_keys = [ - serialize_entity_key( + self._hbase_row_key( entity_key, - entity_key_serialization_version=config.entity_key_serialization_version, - ).hex() + feature_view_name=table.name, + config=config, + ) for entity_key in entity_keys ] rows = hbase.rows(table_name, row_keys=row_keys) @@ -201,17 +199,17 @@ def update( tables_to_delete: Tables to delete from the Hbase Online Store. tables_to_keep: Tables to keep in the Hbase Online Store. """ - hbase = HbaseUtils(self._get_conn(config)) + hbase = HBaseConnector(self._get_conn(config)) project = config.project # We don't create any special state for the entites in this implementation. for table in tables_to_keep: - table_name = _table_id(project, table) + table_name = self._table_id(project, table) if not hbase.check_if_table_exist(table_name): hbase.create_table_with_default_cf(table_name) for table in tables_to_delete: - table_name = _table_id(project, table) + table_name = self._table_id(project, table) hbase.delete_table(table_name) def teardown( @@ -227,20 +225,47 @@ def teardown( config: The RepoConfig for the current FeatureStore. tables: Tables to delete from the feature repo. """ - hbase = HbaseUtils(self._get_conn(config)) + hbase = HBaseConnector(self._get_conn(config)) project = config.project for table in tables: - table_name = _table_id(project, table) + table_name = self._table_id(project, table) hbase.delete_table(table_name) + def _hbase_row_key( + self, + entity_key: EntityKeyProto, + feature_view_name: str, + config: RepoConfig, + ) -> bytes: + """ + Computes the HBase row key for a given entity key and feature view name. -def _table_id(project: str, table: FeatureView) -> str: - """ - Returns table name given the project_name and the feature_view. + Args: + entity_key (EntityKeyProto): The entity key to compute the row key for. + feature_view_name (str): The name of the feature view to compute the row key for. + config (RepoConfig): The configuration for the Feast repository. - Args: - project: Name of the feast project. - table: Feast FeatureView. - """ - return f"{project}_{table.name}" + Returns: + bytes: The HBase row key for the given entity key and feature view name. + """ + entity_id = compute_entity_id( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ) + # Even though `entity_id` uniquely identifies an entity, we use the same table + # for multiple feature_views with the same set of entities. + # To uniquely identify the row for a feature_view, we suffix the name of the feature_view itself. + # This also ensures that features for entities from various feature_views are + # colocated. + return f"{entity_id}#{feature_view_name}".encode() + + def _table_id(self, project: str, table: FeatureView) -> str: + """ + Returns table name given the project_name and the feature_view. + + Args: + project: Name of the feast project. + table: Feast FeatureView. + """ + return f"{project}:{table.name}" diff --git a/sdk/python/feast/infra/online_stores/redis.py b/sdk/python/feast/infra/online_stores/redis.py index 83922068ac..9561705aaa 100644 --- a/sdk/python/feast/infra/online_stores/redis.py +++ b/sdk/python/feast/infra/online_stores/redis.py @@ -43,6 +43,7 @@ try: from redis import Redis from redis.cluster import ClusterNode, RedisCluster + from redis.sentinel import Sentinel except ImportError as e: from feast.errors import FeastExtrasDependencyImportError @@ -54,6 +55,7 @@ class RedisType(str, Enum): redis = "redis" redis_cluster = "redis_cluster" + redis_sentinel = "redis_sentinel" class RedisOnlineStoreConfig(FeastConfigBaseModel): @@ -65,6 +67,9 @@ class RedisOnlineStoreConfig(FeastConfigBaseModel): redis_type: RedisType = RedisType.redis """Redis type: redis or redis_cluster""" + sentinel_master: StrictStr = "mymaster" + """Sentinel's master name""" + connection_string: StrictStr = "localhost:6379" """Connection string containing the host, port, and configuration parameters for Redis format: host:port,parameter1,parameter2 eg. redis:6379,db=0 """ @@ -178,6 +183,15 @@ def _get_client(self, online_store_config: RedisOnlineStoreConfig): ClusterNode(**node) for node in startup_nodes ] self._client = RedisCluster(**kwargs) + elif online_store_config.redis_type == RedisType.redis_sentinel: + sentinel_hosts = [] + + for item in startup_nodes: + sentinel_hosts.append((item["host"], int(item["port"]))) + + sentinel = Sentinel(sentinel_hosts, **kwargs) + master = sentinel.master_for(online_store_config.sentinel_master) + self._client = master else: kwargs["host"] = startup_nodes[0]["host"] kwargs["port"] = startup_nodes[0]["port"] diff --git a/sdk/python/feast/infra/registry/proto_registry_utils.py b/sdk/python/feast/infra/registry/proto_registry_utils.py index c7eeea0f82..e93f513b69 100644 --- a/sdk/python/feast/infra/registry/proto_registry_utils.py +++ b/sdk/python/feast/infra/registry/proto_registry_utils.py @@ -1,4 +1,5 @@ import uuid +from functools import wraps from typing import List, Optional from feast import usage @@ -23,6 +24,26 @@ from feast.stream_feature_view import StreamFeatureView +def registry_proto_cache(func): + cache_key = None + cache_value = None + + @wraps(func) + def wrapper(registry_proto: RegistryProto, project: str): + nonlocal cache_key, cache_value + + key = tuple([id(registry_proto), registry_proto.version_id, project]) + + if key == cache_key: + return cache_value + else: + cache_value = func(registry_proto, project) + cache_key = key + return cache_value + + return wrapper + + def init_project_metadata(cached_registry_proto: RegistryProto, project: str): new_project_uuid = f"{uuid.uuid4()}" usage.set_current_project_uuid(new_project_uuid) @@ -137,8 +158,9 @@ def get_validation_reference( raise ValidationReferenceNotFound(name, project=project) +@registry_proto_cache def list_feature_services( - registry_proto: RegistryProto, project: str, allow_cache: bool = False + registry_proto: RegistryProto, project: str ) -> List[FeatureService]: feature_services = [] for feature_service_proto in registry_proto.feature_services: @@ -147,6 +169,7 @@ def list_feature_services( return feature_services +@registry_proto_cache def list_feature_views( registry_proto: RegistryProto, project: str ) -> List[FeatureView]: @@ -157,6 +180,7 @@ def list_feature_views( return feature_views +@registry_proto_cache def list_request_feature_views( registry_proto: RegistryProto, project: str ) -> List[RequestFeatureView]: @@ -169,6 +193,7 @@ def list_request_feature_views( return feature_views +@registry_proto_cache def list_stream_feature_views( registry_proto: RegistryProto, project: str ) -> List[StreamFeatureView]: @@ -181,6 +206,7 @@ def list_stream_feature_views( return stream_feature_views +@registry_proto_cache def list_on_demand_feature_views( registry_proto: RegistryProto, project: str ) -> List[OnDemandFeatureView]: @@ -193,6 +219,7 @@ def list_on_demand_feature_views( return on_demand_feature_views +@registry_proto_cache def list_entities(registry_proto: RegistryProto, project: str) -> List[Entity]: entities = [] for entity_proto in registry_proto.entities: @@ -201,6 +228,7 @@ def list_entities(registry_proto: RegistryProto, project: str) -> List[Entity]: return entities +@registry_proto_cache def list_data_sources(registry_proto: RegistryProto, project: str) -> List[DataSource]: data_sources = [] for data_source_proto in registry_proto.data_sources: @@ -209,6 +237,7 @@ def list_data_sources(registry_proto: RegistryProto, project: str) -> List[DataS return data_sources +@registry_proto_cache def list_saved_datasets( registry_proto: RegistryProto, project: str ) -> List[SavedDataset]: @@ -219,6 +248,7 @@ def list_saved_datasets( return saved_datasets +@registry_proto_cache def list_validation_references( registry_proto: RegistryProto, project: str ) -> List[ValidationReference]: @@ -231,6 +261,7 @@ def list_validation_references( return validation_references +@registry_proto_cache def list_project_metadata( registry_proto: RegistryProto, project: str ) -> List[ProjectMetadata]: diff --git a/sdk/python/feast/infra/utils/aws_utils.py b/sdk/python/feast/infra/utils/aws_utils.py index f48dfbb86b..728bcab791 100644 --- a/sdk/python/feast/infra/utils/aws_utils.py +++ b/sdk/python/feast/infra/utils/aws_utils.py @@ -351,7 +351,9 @@ def upload_arrow_table_to_redshift( else: # Write the PyArrow Table on disk in Parquet format and upload it to S3 with tempfile.TemporaryFile(suffix=".parquet") as parquet_temp_file: - pq.write_table(table, parquet_temp_file) + # In Pyarrow v13.0, the parquet version was upgraded to v2.6 from v2.4. + # Set the coerce_timestamps to "us"(microseconds) for backward compatibility. + pq.write_table(table, parquet_temp_file, coerce_timestamps="us") parquet_temp_file.seek(0) s3_resource.Object(bucket, key).put(Body=parquet_temp_file) diff --git a/sdk/python/feast/infra/utils/hbase_utils.py b/sdk/python/feast/infra/utils/hbase_utils.py index 4816a60087..d44f93f161 100644 --- a/sdk/python/feast/infra/utils/hbase_utils.py +++ b/sdk/python/feast/infra/utils/hbase_utils.py @@ -1,9 +1,6 @@ from typing import List -from happybase import Connection - -from feast.infra.key_encoding_utils import serialize_entity_key -from feast.protos.feast.types.EntityKey_pb2 import EntityKey +from happybase import ConnectionPool class HbaseConstants: @@ -28,7 +25,7 @@ def get_col_from_feature(feature): return HbaseConstants.DEFAULT_COLUMN_FAMILY + ":" + feature -class HbaseUtils: +class HBaseConnector: """ Utils class to manage different Hbase operations. @@ -40,14 +37,22 @@ class HbaseUtils: """ def __init__( - self, conn: Connection = None, host: str = None, port: int = None, timeout=None + self, + pool: ConnectionPool = None, + host: str = None, + port: int = None, + connection_pool_size: int = 4, ): - if conn is None: + if pool is None: self.host = host self.port = port - self.conn = Connection(host=host, port=port, timeout=timeout) + self.pool = ConnectionPool( + host=host, + port=port, + size=connection_pool_size, + ) else: - self.conn = conn + self.pool = pool def create_table(self, table_name: str, colm_family: List[str]): """ @@ -60,7 +65,9 @@ def create_table(self, table_name: str, colm_family: List[str]): cf_dict: dict = {} for cf in colm_family: cf_dict[cf] = dict() - return self.conn.create_table(table_name, cf_dict) + + with self.pool.connection() as conn: + return conn.create_table(table_name, cf_dict) def create_table_with_default_cf(self, table_name: str): """ @@ -69,7 +76,8 @@ def create_table_with_default_cf(self, table_name: str): Arguments: table_name: Name of the Hbase table. """ - return self.conn.create_table(table_name, {"default": dict()}) + with self.pool.connection() as conn: + return conn.create_table(table_name, {"default": dict()}) def check_if_table_exist(self, table_name: str): """ @@ -78,16 +86,18 @@ def check_if_table_exist(self, table_name: str): Arguments: table_name: Name of the Hbase table. """ - return bytes(table_name, "utf-8") in self.conn.tables() + with self.pool.connection() as conn: + return bytes(table_name, "utf-8") in conn.tables() def batch(self, table_name: str): """ - Returns a 'Batch' instance that can be used for mass data manipulation in the hbase table. + Returns a "Batch" instance that can be used for mass data manipulation in the hbase table. Arguments: table_name: Name of the Hbase table. """ - return self.conn.table(table_name).batch() + with self.pool.connection() as conn: + return conn.table(table_name).batch() def put(self, table_name: str, row_key: str, data: dict): """ @@ -98,8 +108,9 @@ def put(self, table_name: str, row_key: str, data: dict): row_key: Row key of the row to be inserted to hbase table. data: Mapping of column family name:column name to column values """ - table = self.conn.table(table_name) - table.put(row_key, data) + with self.pool.connection() as conn: + table = conn.table(table_name) + table.put(row_key, data) def row( self, @@ -119,8 +130,9 @@ def row( timestamp: timestamp specifies the maximum version the cells can have. include_timestamp: specifies if (column, timestamp) to be return instead of only column. """ - table = self.conn.table(table_name) - return table.row(row_key, columns, timestamp, include_timestamp) + with self.pool.connection() as conn: + table = conn.table(table_name) + return table.row(row_key, columns, timestamp, include_timestamp) def rows( self, @@ -140,52 +152,69 @@ def rows( timestamp: timestamp specifies the maximum version the cells can have. include_timestamp: specifies if (column, timestamp) to be return instead of only column. """ - table = self.conn.table(table_name) - return table.rows(row_keys, columns, timestamp, include_timestamp) + with self.pool.connection() as conn: + table = conn.table(table_name) + return table.rows(row_keys, columns, timestamp, include_timestamp) def print_table(self, table_name): """Prints the table scanning all the rows of the hbase table.""" - table = self.conn.table(table_name) - scan_data = table.scan() - for row_key, cols in scan_data: - print(row_key.decode("utf-8"), cols) + with self.pool.connection() as conn: + table = conn.table(table_name) + scan_data = table.scan() + for row_key, cols in scan_data: + print(row_key.decode("utf-8"), cols) def delete_table(self, table: str): """Deletes the hbase table given the table name.""" if self.check_if_table_exist(table): - self.conn.delete_table(table, disable=True) + with self.pool.connection() as conn: + conn.delete_table(table, disable=True) def close_conn(self): """Closes the happybase connection.""" - self.conn.close() + with self.pool.connection() as conn: + conn.close() def main(): + from feast.infra.key_encoding_utils import serialize_entity_key + from feast.protos.feast.types.EntityKey_pb2 import EntityKey from feast.protos.feast.types.Value_pb2 import Value - connection = Connection(host="localhost", port=9090) - table = connection.table("test_hbase_driver_hourly_stats") - row_keys = [ - serialize_entity_key( - EntityKey(join_keys=["driver_id"], entity_values=[Value(int64_val=1004)]), - entity_key_serialization_version=2, - ).hex(), - serialize_entity_key( - EntityKey(join_keys=["driver_id"], entity_values=[Value(int64_val=1005)]), - entity_key_serialization_version=2, - ).hex(), - serialize_entity_key( - EntityKey(join_keys=["driver_id"], entity_values=[Value(int64_val=1024)]), - entity_key_serialization_version=2, - ).hex(), - ] - rows = table.rows(row_keys) - - for row_key, row in rows: - for key, value in row.items(): - col_name = bytes.decode(key, "utf-8").split(":")[1] - print(col_name, value) - print() + pool = ConnectionPool( + host="localhost", + port=9090, + size=2, + ) + with pool.connection() as connection: + table = connection.table("test_hbase_driver_hourly_stats") + row_keys = [ + serialize_entity_key( + EntityKey( + join_keys=["driver_id"], entity_values=[Value(int64_val=1004)] + ), + entity_key_serialization_version=2, + ).hex(), + serialize_entity_key( + EntityKey( + join_keys=["driver_id"], entity_values=[Value(int64_val=1005)] + ), + entity_key_serialization_version=2, + ).hex(), + serialize_entity_key( + EntityKey( + join_keys=["driver_id"], entity_values=[Value(int64_val=1024)] + ), + entity_key_serialization_version=2, + ).hex(), + ] + rows = table.rows(row_keys) + + for _, row in rows: + for key, value in row.items(): + col_name = bytes.decode(key, "utf-8").split(":")[1] + print(col_name, value) + print() if __name__ == "__main__": diff --git a/sdk/python/feast/infra/utils/postgres/postgres_config.py b/sdk/python/feast/infra/utils/postgres/postgres_config.py index 9fbaed474d..a4ebb456ef 100644 --- a/sdk/python/feast/infra/utils/postgres/postgres_config.py +++ b/sdk/python/feast/infra/utils/postgres/postgres_config.py @@ -25,4 +25,4 @@ class PostgreSQLConfig(FeastConfigBaseModel): sslkey_path: Optional[StrictStr] = None sslcert_path: Optional[StrictStr] = None sslrootcert_path: Optional[StrictStr] = None - keepalives_idle: int = 0 + keepalives_idle: Optional[int] = None diff --git a/sdk/python/feast/proto_json.py b/sdk/python/feast/proto_json.py index a0a4dce86b..41d2afa55a 100644 --- a/sdk/python/feast/proto_json.py +++ b/sdk/python/feast/proto_json.py @@ -1,13 +1,13 @@ import uuid from typing import Any, Callable, Type -import pkg_resources from google.protobuf.json_format import ( # type: ignore _WKTJSONMETHODS, ParseError, _Parser, _Printer, ) +from importlib_metadata import version as importlib_version from packaging import version from feast.protos.feast.serving.ServingService_pb2 import FeatureList @@ -118,7 +118,7 @@ def from_json_object_updated( # https://github.com/feast-dev/feast/issues/2484 Certain feast users need a higher version of protobuf but the # parameters of `from_json_object` changes in feast 3.20.1. This change gives users flexibility to use earlier versions. - current_version = pkg_resources.get_distribution("protobuf").version + current_version = importlib_version("protobuf") if version.parse(current_version) < version.parse("3.20"): _patch_proto_json_encoding(Value, to_json_object, from_json_object) else: @@ -168,7 +168,7 @@ def from_json_object( # https://github.com/feast-dev/feast/issues/2484 Certain feast users need a higher version of protobuf but the # parameters of `from_json_object` changes in feast 3.20.1. This change gives users flexibility to use earlier versions. - current_version = pkg_resources.get_distribution("protobuf").version + current_version = importlib_version("protobuf") if version.parse(current_version) < version.parse("3.20"): _patch_proto_json_encoding(RepeatedValue, to_json_object, from_json_object) else: @@ -221,7 +221,7 @@ def from_json_object_updated( # https://github.com/feast-dev/feast/issues/2484 Certain feast users need a higher version of protobuf but the # parameters of `from_json_object` changes in feast 3.20.1. This change gives users flexibility to use earlier versions. - current_version = pkg_resources.get_distribution("protobuf").version + current_version = importlib_version("protobuf") if version.parse(current_version) < version.parse("3.20"): _patch_proto_json_encoding(FeatureList, to_json_object, from_json_object) else: diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py index 3f49069066..cdb65f886e 100644 --- a/sdk/python/feast/type_map.py +++ b/sdk/python/feast/type_map.py @@ -428,12 +428,15 @@ def _python_value_to_proto_value( for value in values ] if feast_value_type in PYTHON_SCALAR_VALUE_TYPE_TO_PROTO_VALUE: - return [ - ProtoValue(**{field_name: func(value)}) - if not pd.isnull(value) - else ProtoValue() - for value in values - ] + out = [] + for value in values: + if isinstance(value, ProtoValue): + out.append(value) + elif not pd.isnull(value): + out.append(ProtoValue(**{field_name: func(value)})) + else: + out.append(ProtoValue()) + return out raise Exception(f"Unsupported data type: ${str(type(values[0]))}") @@ -746,7 +749,7 @@ def spark_to_feast_value_type(spark_type_as_str: str) -> ValueType: "array": ValueType.UNIX_TIMESTAMP_LIST, } # TODO: Find better way of doing this. - if type(spark_type_as_str) != str or spark_type_as_str not in type_map: + if not isinstance(spark_type_as_str, str) or spark_type_as_str not in type_map: return ValueType.NULL return type_map[spark_type_as_str.lower()] diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 80956440ab..1d2951cf57 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -6,7 +6,7 @@ "@elastic/datemath": "^5.0.3", "@elastic/eui": "^55.0.1", "@emotion/react": "^11.9.0", - "@feast-dev/feast-ui": "0.34.0", + "@feast-dev/feast-ui": "0.35.0", "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.2.0", "@testing-library/user-event": "^13.5.0", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 6fb431576a..4d6f690b94 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -26,12 +26,13 @@ dependencies: "@babel/highlight" "^7.16.7" -"@babel/code-frame@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.5.tgz#234d98e1551960604f1246e6475891a570ad5658" - integrity sha512-Xmwn266vad+6DAqEB2A6V/CcZVp62BbwVmcOJc2RPuwih1kw02TjQvWVWlcKGbBPd+8/0V5DEkOcizRGYsspYQ== +"@babel/code-frame@^7.22.13": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== dependencies: - "@babel/highlight" "^7.22.5" + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" "@babel/compat-data@^7.13.11", "@babel/compat-data@^7.17.0", "@babel/compat-data@^7.17.10": version "7.17.10" @@ -77,12 +78,12 @@ "@jridgewell/gen-mapping" "^0.1.0" jsesc "^2.5.1" -"@babel/generator@^7.22.7": - version "7.22.9" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.9.tgz#572ecfa7a31002fa1de2a9d91621fd895da8493d" - integrity sha512-KtLMbmicyuK2Ak/FTCJVbDnkN1SlT8/kceFTiuDiiRUUSMnHMidxSCdG4ndkTOHHpoomWe/4xkvHkEOncwjYIw== +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== dependencies: - "@babel/types" "^7.22.5" + "@babel/types" "^7.23.0" "@jridgewell/gen-mapping" "^0.3.2" "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" @@ -161,10 +162,10 @@ dependencies: "@babel/types" "^7.16.7" -"@babel/helper-environment-visitor@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz#f06dd41b7c1f44e1f8da6c4055b41ab3a09a7e98" - integrity sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q== +"@babel/helper-environment-visitor@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" + integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== "@babel/helper-explode-assignable-expression@^7.16.7": version "7.16.7" @@ -181,13 +182,13 @@ "@babel/template" "^7.16.7" "@babel/types" "^7.17.0" -"@babel/helper-function-name@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be" - integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ== +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== dependencies: - "@babel/template" "^7.22.5" - "@babel/types" "^7.22.5" + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" "@babel/helper-hoist-variables@^7.16.7": version "7.16.7" @@ -313,6 +314,11 @@ resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== +"@babel/helper-validator-identifier@^7.22.20": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz#c4ae002c61d2879e724581d96665583dbc1dc0e0" + integrity sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A== + "@babel/helper-validator-identifier@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" @@ -351,13 +357,13 @@ chalk "^2.0.0" js-tokens "^4.0.0" -"@babel/highlight@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.5.tgz#aa6c05c5407a67ebce408162b7ede789b4d22031" - integrity sha512-BSKlD1hgnedS5XRnGOljZawtag7H1yPfQp0tdNJCHoH6AZ+Pcm9VvkrK59/Yy593Ypg0zMxH2BxD1VPYUQ7UIw== +"@babel/highlight@^7.22.13": + version "7.22.20" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.20.tgz#4ca92b71d80554b01427815e06f2df965b9c1f54" + integrity sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg== dependencies: - "@babel/helper-validator-identifier" "^7.22.5" - chalk "^2.0.0" + "@babel/helper-validator-identifier" "^7.22.20" + chalk "^2.4.2" js-tokens "^4.0.0" "@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.16.7", "@babel/parser@^7.17.10": @@ -365,10 +371,10 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.17.10.tgz#873b16db82a8909e0fbd7f115772f4b739f6ce78" integrity sha512-n2Q6i+fnJqzOaq2VkdXxy2TCPCWQZHiCo0XqmrCvDWcZQKRyZzYi4Z0yxlBuN0w+r2ZHmre+Q087DSrw3pbJDQ== -"@babel/parser@^7.22.5", "@babel/parser@^7.22.7": - version "7.22.7" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.7.tgz#df8cf085ce92ddbdbf668a7f186ce848c9036cae" - integrity sha512-7NF8pOkHP5o2vpmGgNGcfAeCvOYhGLyA3Z4eBQkT1RJlWu47n63bCs93QfJ2hIAFCil7L5P2IWhs1oToVgrL0Q== +"@babel/parser@^7.22.15", "@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== "@babel/plugin-bugfix-safari-id-destructuring-collision-in-function-expression@^7.16.7": version "7.16.7" @@ -1135,44 +1141,28 @@ "@babel/parser" "^7.16.7" "@babel/types" "^7.16.7" -"@babel/template@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.5.tgz#0c8c4d944509875849bd0344ff0050756eefc6ec" - integrity sha512-X7yV7eiwAxdj9k94NEylvbVHLiVG1nvzCV2EAowhxLTwODV1jl9UzZ48leOC0sH7OnuHrIkllaBgneUykIcZaw== +"@babel/template@^7.22.15": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" + integrity sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w== dependencies: - "@babel/code-frame" "^7.22.5" - "@babel/parser" "^7.22.5" - "@babel/types" "^7.22.5" - -"@babel/traverse@^7.13.0", "@babel/traverse@^7.16.7", "@babel/traverse@^7.16.8", "@babel/traverse@^7.17.10", "@babel/traverse@^7.17.3", "@babel/traverse@^7.17.9", "@babel/traverse@^7.7.2": - version "7.17.10" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.17.10.tgz#1ee1a5ac39f4eac844e6cf855b35520e5eb6f8b5" - integrity sha512-VmbrTHQteIdUUQNTb+zE12SHS/xQVIShmBPhlNP12hD5poF2pbITW1Z4172d03HegaQWhLffdkRJYtAzp0AGcw== - dependencies: - "@babel/code-frame" "^7.16.7" - "@babel/generator" "^7.17.10" - "@babel/helper-environment-visitor" "^7.16.7" - "@babel/helper-function-name" "^7.17.9" - "@babel/helper-hoist-variables" "^7.16.7" - "@babel/helper-split-export-declaration" "^7.16.7" - "@babel/parser" "^7.17.10" - "@babel/types" "^7.17.10" - debug "^4.1.0" - globals "^11.1.0" + "@babel/code-frame" "^7.22.13" + "@babel/parser" "^7.22.15" + "@babel/types" "^7.22.15" -"@babel/traverse@^7.4.5": - version "7.22.8" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.8.tgz#4d4451d31bc34efeae01eac222b514a77aa4000e" - integrity sha512-y6LPR+wpM2I3qJrsheCTwhIinzkETbplIgPBbwvqPKc+uljeA5gP+3nP8irdYt1mjQaDnlIcG+dw8OjAco4GXw== +"@babel/traverse@^7.13.0", "@babel/traverse@^7.16.7", "@babel/traverse@^7.16.8", "@babel/traverse@^7.17.10", "@babel/traverse@^7.17.3", "@babel/traverse@^7.17.9", "@babel/traverse@^7.4.5", "@babel/traverse@^7.7.2": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== dependencies: - "@babel/code-frame" "^7.22.5" - "@babel/generator" "^7.22.7" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.0" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" "@babel/helper-hoist-variables" "^7.22.5" "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.7" - "@babel/types" "^7.22.5" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -1184,6 +1174,15 @@ "@babel/helper-validator-identifier" "^7.16.7" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15", "@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@babel/types@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.5.tgz#cd93eeaab025880a3a47ec881f4b096a5b786fbe" @@ -1452,10 +1451,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@feast-dev/feast-ui@0.34.0": - version "0.34.0" - resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.34.0.tgz#21799c119da936814ff47e1d1d297258ecfda098" - integrity sha512-ErrnmaFPMgnmnTpBpn7T0oiC3cA+atE5yKBmoMFXBsK6aplu18aF53QBX4JJFKMKF+A2BcO37veeXm1k01SJQQ== +"@feast-dev/feast-ui@0.35.0": + version "0.35.0" + resolved "https://registry.yarnpkg.com/@feast-dev/feast-ui/-/feast-ui-0.35.0.tgz#f28eb82ae4855673230f14e3740a7786f545fdd7" + integrity sha512-t0Rd2TWUMim6ITfVVlQU8aBZboLvxla6Z7udGW+tQ3UUGcq1VbM6/y+GobuYQfbdHHeF2GmlYqp6zw5DuoIM+Q== dependencies: "@elastic/datemath" "^5.0.3" "@elastic/eui" "^55.0.1" @@ -3690,7 +3689,7 @@ ccount@^1.0.0: resolved "https://registry.yarnpkg.com/ccount/-/ccount-1.1.0.tgz#246687debb6014735131be8abab2d93898f8d043" integrity sha512-vlNK021QdI7PNeiUh/lKkC/mNHHfV0m/Ad5JoI0TYtlBnJAslM/JIkm/tGC88bkLIwO6OQ5uV6ztS6kVAtCDlg== -chalk@^2.0.0, chalk@^2.4.1: +chalk@^2.0.0, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -5486,9 +5485,9 @@ focus-lock@^0.11.2: tslib "^2.0.3" follow-redirects@^1.0.0: - version "1.15.0" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.0.tgz#06441868281c86d0dda4ad8bdaead2d02dca89d4" - integrity sha512-aExlJShTV4qOUOL7yF1U5tvLCB0xQuudbf6toyYA0E/acBNw71mvjFTnLaRp50aQaYocMR0a/RMMBIHeZnGyjQ== + version "1.15.4" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" + integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.2" diff --git a/sdk/python/feast/ui_server.py b/sdk/python/feast/ui_server.py index e750f280ad..8a39293f91 100644 --- a/sdk/python/feast/ui_server.py +++ b/sdk/python/feast/ui_server.py @@ -2,7 +2,7 @@ import threading from typing import Callable, Optional -import pkg_resources +import importlib_resources import uvicorn from fastapi import FastAPI, Response from fastapi.middleware.cors import CORSMiddleware @@ -51,20 +51,21 @@ def shutdown_event(): async_refresh() - ui_dir = pkg_resources.resource_filename(__name__, "ui/build/") - # Initialize with the projects-list.json file - with open(ui_dir + "projects-list.json", mode="w") as f: - projects_dict = { - "projects": [ - { - "name": "Project", - "description": "Test project", - "id": project_id, - "registryPath": f"{root_path}/registry", - } - ] - } - f.write(json.dumps(projects_dict)) + ui_dir_ref = importlib_resources.files(__name__) / "ui/build/" + with importlib_resources.as_file(ui_dir_ref) as ui_dir: + # Initialize with the projects-list.json file + with ui_dir.joinpath("projects-list.json").open(mode="w") as f: + projects_dict = { + "projects": [ + { + "name": "Project", + "description": "Test project", + "id": project_id, + "registryPath": f"{root_path}/registry", + } + ] + } + f.write(json.dumps(projects_dict)) @app.get("/registry") def read_registry(): @@ -76,7 +77,7 @@ def read_registry(): # For all other paths (such as paths that would otherwise be handled by react router), pass to React @app.api_route("/p/{path_name:path}", methods=["GET"]) def catch_all(): - filename = ui_dir + "index.html" + filename = ui_dir.joinpath("index.html") with open(filename) as f: content = f.read() diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index a59553b4ac..2cbfa6cbc9 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,93 +4,64 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.10-ci-requirements.txt # -adal==1.2.7 - # via msrestazure -adlfs==0.5.9 - # via feast (setup.py) -aiohttp==3.8.5 - # via - # adlfs - # gcsfs -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations anyio==4.0.0 # via - # httpcore + # httpx # jupyter-server # starlette # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.3 - # via - # ipykernel - # ipython argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.2.3 +arrow==1.3.0 # via isoduration asn1crypto==1.5.1 - # via - # oscrypto - # snowflake-connector-python + # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.0 +asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 - # via - # aiohttp - # redis + # via redis attrs==23.1.0 # via - # aiohttp # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.3 +azure-core==1.29.5 # via - # adlfs # azure-identity # azure-storage-blob - # msrest -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via - # adlfs - # feast (setup.py) -azure-storage-blob==12.17.0 - # via - # adlfs - # feast (setup.py) -babel==2.12.1 +azure-identity==1.15.0 + # via feast (setup.py) +azure-storage-blob==12.19.0 + # via feast (setup.py) +babel==2.13.1 # via # jupyterlab-server # sphinx -backcall==0.2.0 - # via ipython beautifulsoup4==4.12.2 # via nbconvert black==22.12.0 # via feast (setup.py) -bleach==6.0.0 +bleach==6.1.0 # via nbconvert -boto3==1.28.42 +boto3==1.29.2 # via # feast (setup.py) # moto -botocore==1.31.42 +botocore==1.32.2 # via # boto3 # moto @@ -105,7 +76,7 @@ bytewax==0.15.1 # via feast (setup.py) cachecontrol==0.13.1 # via firebase-admin -cachetools==5.3.1 +cachetools==5.3.2 # via google-auth cassandra-driver==3.28.0 # via feast (setup.py) @@ -115,20 +86,17 @@ certifi==2023.7.22 # httpx # kubernetes # minio - # msrest # requests # snowflake-connector-python -cffi==1.15.1 +cffi==1.16.0 # via # argon2-cffi-bindings - # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.4.0 # via pre-commit -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via - # aiohttp # requests # snowflake-connector-python click==8.1.7 @@ -142,21 +110,22 @@ click==8.1.7 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.4 +comm==0.2.0 # via # ipykernel # ipywidgets -coverage[toml]==7.3.1 - # via pytest-cov -cryptography==41.0.3 +coverage[toml]==7.3.2 + # via + # coverage + # pytest-cov +cryptography==41.0.5 # via - # adal # azure-identity # azure-storage-blob # feast (setup.py) @@ -168,16 +137,14 @@ cryptography==41.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.9.1 +dask==2023.11.0 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7.post1 +debugpy==1.8.0 # via ipykernel decorator==5.1.1 - # via - # gcsfs - # ipython + # via ipython defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 @@ -204,17 +171,17 @@ exceptiongroup==1.1.3 # pytest execnet==2.0.2 # via pytest-xdist -executing==1.2.0 +executing==2.0.1 # via stack-data fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro -fastjsonschema==2.18.0 +fastjsonschema==2.19.0 # via nbformat -filelock==3.12.3 +filelock==3.13.1 # via # snowflake-connector-python # virtualenv @@ -226,22 +193,15 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.4.0 +fsspec==2023.9.2 # via - # aiohttp - # aiosignal -fsspec==2022.1.0 - # via - # adlfs # dask - # gcsfs -gcsfs==2022.1.0 - # via feast (setup.py) + # feast (setup.py) geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.1 +google-api-core[grpc]==2.14.0 # via # feast (setup.py) # firebase-admin @@ -253,24 +213,22 @@ google-api-core[grpc]==2.11.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.98.0 +google-api-python-client==2.108.0 # via firebase-admin -google-auth==2.22.0 +google-auth==2.23.4 # via - # gcsfs # google-api-core # google-api-python-client # google-auth-httplib2 - # google-auth-oauthlib # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.0 +google-auth-httplib2==0.1.1 # via google-api-python-client -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-bigquery[pandas]==3.11.4 - # via feast (setup.py) +google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) google-cloud-bigtable==2.21.0 @@ -284,20 +242,21 @@ google-cloud-core==2.3.3 # google-cloud-storage google-cloud-datastore==2.18.0 # via feast (setup.py) -google-cloud-firestore==2.11.1 +google-cloud-firestore==2.13.1 # via firebase-admin -google-cloud-storage==2.10.0 +google-cloud-storage==2.13.0 # via # feast (setup.py) # firebase-admin - # gcsfs google-crc32c==1.5.0 - # via google-resumable-media + # via + # google-cloud-storage + # google-resumable-media google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.60.0 +googleapis-common-protos[grpc]==1.61.0 # via # feast (setup.py) # google-api-core @@ -305,9 +264,11 @@ googleapis-common-protos[grpc]==1.60.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -grpc-google-iam-v1==0.12.6 +greenlet==3.0.1 + # via sqlalchemy +grpc-google-iam-v1==0.12.7 # via google-cloud-bigtable -grpcio==1.57.0 +grpcio==1.59.2 # via # feast (setup.py) # google-api-core @@ -319,15 +280,15 @@ grpcio==1.57.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-status==1.57.0 +grpcio-status==1.59.2 # via google-api-core -grpcio-testing==1.57.0 +grpcio-testing==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -341,17 +302,17 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.3 +httpcore==1.0.2 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) -identify==2.5.27 +identify==2.5.31 # via pre-commit idna==3.4 # via @@ -360,33 +321,33 @@ idna==3.4 # jsonschema # requests # snowflake-connector-python - # yarl imagesize==1.4.1 # via sphinx importlib-metadata==6.8.0 # via # dask + # feast (setup.py) # great-expectations +importlib-resources==6.1.1 + # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.25.2 +ipykernel==6.26.0 # via jupyterlab -ipython==8.15.0 +ipython==8.17.2 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.0 +ipywidgets==8.1.1 # via great-expectations isodate==0.6.1 - # via - # azure-storage-blob - # msrest + # via azure-storage-blob isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.19.0 +jedi==0.19.1 # via ipython jinja2==3.1.2 # via @@ -411,7 +372,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.19.0 +jsonschema[format-nongpl]==4.20.0 # via # altair # feast (setup.py) @@ -419,14 +380,14 @@ jsonschema[format-nongpl]==4.19.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema -jupyter-client==8.3.1 +jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.3.1 +jupyter-core==5.5.0 # via # ipykernel # jupyter-client @@ -435,11 +396,11 @@ jupyter-core==5.3.1 # nbclient # nbconvert # nbformat -jupyter-events==0.7.0 +jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.0 # via jupyterlab -jupyter-server==2.7.3 +jupyter-server==2.10.1 # via # jupyter-lsp # jupyterlab @@ -448,21 +409,21 @@ jupyter-server==2.7.3 # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server -jupyterlab==4.0.5 +jupyterlab==4.0.8 # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-server==2.24.0 +jupyterlab-server==2.25.1 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.8 +jupyterlab-widgets==3.0.9 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd -makefun==1.15.1 +makefun==1.15.2 # via great-expectations markupsafe==2.1.3 # via @@ -479,7 +440,7 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==3.0.1 +mistune==3.0.2 # via # great-expectations # nbconvert @@ -489,25 +450,16 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.2 +moto==4.2.9 # via feast (setup.py) -msal==1.23.0 +msal==1.25.0 # via - # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.0.0 # via azure-identity -msgpack==1.0.5 +msgpack==1.0.7 # via cachecontrol -msrest==0.7.1 - # via msrestazure -msrestazure==0.6.4 - # via adlfs -multidict==6.0.4 - # via - # aiohttp - # yarl multiprocess==0.70.15 # via bytewax mypy==0.982 @@ -518,13 +470,13 @@ mypy-extensions==1.0.0 # via # black # mypy -mypy-protobuf==3.1 +mypy-protobuf==3.1.0 # via feast (setup.py) mysqlclient==2.2.0 # via feast (setup.py) -nbclient==0.8.0 +nbclient==0.9.0 # via nbconvert -nbconvert==7.8.0 +nbconvert==7.11.0 # via jupyter-server nbformat==5.9.2 # via @@ -532,17 +484,17 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.7 +nest-asyncio==1.5.8 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.3 +notebook==7.0.6 # via great-expectations notebook-shim==0.2.3 # via # jupyterlab # notebook -numpy==1.25.2 +numpy==1.24.4 # via # altair # db-dtypes @@ -554,11 +506,9 @@ numpy==1.25.2 # scipy oauthlib==3.2.2 # via requests-oauthlib -oscrypto==1.3.0 - # via snowflake-connector-python overrides==7.4.0 # via jupyter-server -packaging==23.1 +packaging==23.2 # via # build # dask @@ -592,19 +542,17 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -partd==1.4.0 +partd==1.4.1 # via dask pathspec==0.11.2 # via black -pbr==5.11.1 +pbr==6.0.0 # via mock pexpect==4.8.0 # via ipython -pickleshare==0.7.5 - # via ipython pip-tools==7.3.0 # via feast (setup.py) -platformdirs==3.8.1 +platformdirs==3.11.0 # via # black # jupyter-core @@ -614,13 +562,13 @@ pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.7.0 +portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via jupyter-server -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython proto-plus==1.22.3 # via @@ -652,7 +600,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -666,7 +614,7 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==10.0.1 +pyarrow==14.0.1 # via # db-dtypes # feast (setup.py) @@ -684,9 +632,7 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.18.0 - # via snowflake-connector-python -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -701,16 +647,15 @@ pygments==2.16.1 # sphinx pyjwt[crypto]==2.8.0 # via - # adal # msal # snowflake-connector-python -pymssql==2.2.8 +pymssql==2.2.10 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==4.0.39 +pyodbc==5.0.1 # via feast (setup.py) -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via snowflake-connector-python pyparsing==3.1.1 # via @@ -718,9 +663,9 @@ pyparsing==3.1.1 # httplib2 pyproject-hooks==1.0.0 # via build -pyspark==3.4.1 +pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.1 +pytest==7.4.3 # via # feast (setup.py) # pytest-benchmark @@ -742,11 +687,10 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.3.1 +pytest-xdist==3.4.0 # via feast (setup.py) python-dateutil==2.8.2 # via - # adal # arrow # botocore # google-cloud-bigquery @@ -783,23 +727,19 @@ pyzmq==25.1.1 # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.8.8 +regex==2023.10.3 # via feast (setup.py) requests==2.31.0 # via - # adal - # adlfs # azure-core - # azure-datalake-store # cachecontrol # docker # feast (setup.py) - # gcsfs # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -808,18 +748,14 @@ requests==2.31.0 # kubernetes # moto # msal - # msrest # requests-oauthlib # responses # snowflake-connector-python # sphinx # trino requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes - # msrest -responses==0.23.3 + # via kubernetes +responses==0.24.1 # via moto rfc3339-validator==0.1.4 # via @@ -831,7 +767,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -839,9 +775,9 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -s3transfer==0.6.2 +s3transfer==0.7.0 # via boto3 -scipy==1.11.2 +scipy==1.11.3 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -852,13 +788,10 @@ six==1.16.0 # bleach # cassandra-driver # geomet - # google-auth - # google-auth-httplib2 # happybase # isodate # kubernetes # mock - # msrestazure # pandavro # python-dateutil # rfc3339-validator @@ -866,12 +799,13 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.1.1 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.5.0 + # via + # feast (setup.py) + # snowflake-connector-python sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -896,11 +830,13 @@ sphinxcontrib-qthelp==1.0.6 # via sphinx sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy -stack-data==0.6.2 +stack-data==0.6.3 # via ipython starlette==0.27.0 # via fastapi @@ -908,13 +844,13 @@ tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 # via feast (setup.py) -terminado==0.17.1 +terminado==0.18.0 # via # jupyter-server # jupyter-server-terminals testcontainers==3.7.1 # via feast (setup.py) -thriftpy2==0.4.16 +thriftpy2==0.4.17 # via happybase tinycss2==1.2.1 # via nbconvert @@ -930,7 +866,7 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest -tomlkit==0.12.1 +tomlkit==0.12.3 # via snowflake-connector-python toolz==0.12.0 # via @@ -949,7 +885,7 @@ tqdm==4.66.1 # via # feast (setup.py) # great-expectations -traitlets==5.9.0 +traitlets==5.13.0 # via # comm # ipykernel @@ -964,7 +900,7 @@ traitlets==5.9.0 # nbclient # nbconvert # nbformat -trino==0.326.0 +trino==0.327.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -974,40 +910,39 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.2 +types-pyopenssl==23.3.0.0 # via types-redis types-python-dateutil==2.8.19.14 - # via feast (setup.py) -types-pytz==2023.3.0.1 - # via feast (setup.py) -types-pyyaml==6.0.12.11 # via + # arrow # feast (setup.py) - # responses -types-redis==4.6.0.5 +types-pytz==2023.3.1.1 + # via feast (setup.py) +types-pyyaml==6.0.12.12 # via feast (setup.py) -types-requests==2.31.0.2 +types-redis==4.6.0.10 # via feast (setup.py) -types-setuptools==68.2.0.0 +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==68.2.0.1 # via feast (setup.py) types-tabulate==0.9.0.3 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # async-lru # azure-core # azure-storage-blob # fastapi - # filelock # great-expectations # mypy # pydantic # snowflake-connector-python # sqlalchemy2-stubs # uvicorn -tzlocal==5.0.1 +tzlocal==5.2 # via # great-expectations # trino @@ -1015,12 +950,11 @@ uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # docker # feast (setup.py) - # google-auth # great-expectations # kubernetes # minio @@ -1028,9 +962,11 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 # via @@ -1038,9 +974,9 @@ virtualenv==20.23.0 # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.6 +wcwidth==0.2.10 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -1048,26 +984,24 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.2 +websocket-client==1.6.4 # via # docker # jupyter-server # kubernetes -websockets==11.0.3 +websockets==12.0 # via uvicorn -werkzeug==2.3.7 +werkzeug==3.0.1 # via moto -wheel==0.41.2 +wheel==0.41.3 # via pip-tools -widgetsnbextension==4.0.8 +widgetsnbextension==4.0.9 # via ipywidgets -wrapt==1.15.0 +wrapt==1.16.0 # via testcontainers xmltodict==0.13.0 # via moto -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 4140bea9d0..0508614aa6 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -6,7 +6,7 @@ # anyio==4.0.0 # via - # httpcore + # httpx # starlette # watchfiles appdirs==1.4.4 @@ -23,7 +23,7 @@ certifi==2023.7.22 # httpcore # httpx # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -32,11 +32,11 @@ click==8.1.7 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.9.1 +dask==2023.11.0 # via feast (setup.py) dill==0.3.7 # via feast (setup.py) @@ -44,25 +44,27 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.9.0 +fsspec==2023.10.0 # via dask -grpcio==1.57.0 +greenlet==3.0.1 + # via sqlalchemy +grpcio==1.59.2 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -70,11 +72,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.3 +httpcore==1.0.2 # via httpx -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) idna==3.4 # via @@ -82,12 +84,16 @@ idna==3.4 # httpx # requests importlib-metadata==6.8.0 - # via dask + # via + # dask + # feast (setup.py) +importlib-resources==6.1.1 + # via feast (setup.py) jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.19.0 +jsonschema==4.20.0 # via feast (setup.py) -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema locket==1.0.0 # via partd @@ -97,19 +103,19 @@ mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.5.1 +mypy==1.7.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -mypy-protobuf==3.1 +mypy-protobuf==3.1.0 # via feast (setup.py) -numpy==1.25.2 +numpy==1.24.4 # via # feast (setup.py) # pandas # pandavro # pyarrow -packaging==23.1 +packaging==23.2 # via # dask # gunicorn @@ -119,7 +125,7 @@ pandas==1.5.3 # pandavro pandavro==1.5.2 # via feast (setup.py) -partd==1.4.0 +partd==1.4.1 # via dask proto-plus==1.22.3 # via feast (setup.py) @@ -131,9 +137,9 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==11.0.0 +pyarrow==14.0.1 # via feast (setup.py) -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -150,13 +156,13 @@ pyyaml==6.0.1 # dask # feast (setup.py) # uvicorn -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -167,11 +173,12 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy starlette==0.27.0 # via fastapi @@ -191,28 +198,30 @@ tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.1 +types-protobuf==4.24.0.4 # via mypy-protobuf -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # fastapi # mypy # pydantic # sqlalchemy2-stubs # uvicorn -urllib3==2.0.4 +urllib3==2.1.0 # via requests -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -websockets==11.0.3 +websockets==12.0 # via uvicorn -zipp==3.16.2 +zipp==3.17.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/sdk/python/requirements/py3.8-ci-requirements.txt b/sdk/python/requirements/py3.8-ci-requirements.txt index b24172e890..8449af824f 100644 --- a/sdk/python/requirements/py3.8-ci-requirements.txt +++ b/sdk/python/requirements/py3.8-ci-requirements.txt @@ -4,77 +4,50 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.8-ci-requirements.txt # -adal==1.2.7 - # via msrestazure -adlfs==0.5.9 - # via feast (setup.py) -aiohttp==3.8.5 - # via - # adlfs - # gcsfs -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations anyio==4.0.0 # via - # httpcore + # httpx # jupyter-server # starlette # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.3 - # via - # ipykernel - # ipython argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.2.3 +arrow==1.3.0 # via isoduration asn1crypto==1.5.1 - # via - # oscrypto - # snowflake-connector-python + # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.0 +asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 - # via - # aiohttp - # redis + # via redis attrs==23.1.0 # via - # aiohttp # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.3 +azure-core==1.29.5 # via - # adlfs # azure-identity # azure-storage-blob - # msrest -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via - # adlfs - # feast (setup.py) -azure-storage-blob==12.17.0 - # via - # adlfs - # feast (setup.py) -babel==2.12.1 +azure-identity==1.15.0 + # via feast (setup.py) +azure-storage-blob==12.19.0 + # via feast (setup.py) +babel==2.13.1 # via # jupyterlab-server # sphinx @@ -88,13 +61,13 @@ beautifulsoup4==4.12.2 # via nbconvert black==22.12.0 # via feast (setup.py) -bleach==6.0.0 +bleach==6.1.0 # via nbconvert -boto3==1.28.42 +boto3==1.29.2 # via # feast (setup.py) # moto -botocore==1.31.42 +botocore==1.32.2 # via # boto3 # moto @@ -109,7 +82,7 @@ bytewax==0.15.1 # via feast (setup.py) cachecontrol==0.13.1 # via firebase-admin -cachetools==5.3.1 +cachetools==5.3.2 # via google-auth cassandra-driver==3.28.0 # via feast (setup.py) @@ -119,20 +92,17 @@ certifi==2023.7.22 # httpx # kubernetes # minio - # msrest # requests # snowflake-connector-python -cffi==1.15.1 +cffi==1.16.0 # via # argon2-cffi-bindings - # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.4.0 # via pre-commit -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via - # aiohttp # requests # snowflake-connector-python click==8.1.7 @@ -146,21 +116,22 @@ click==8.1.7 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.4 +comm==0.2.0 # via # ipykernel # ipywidgets -coverage[toml]==7.3.1 - # via pytest-cov -cryptography==41.0.3 +coverage[toml]==7.3.2 + # via + # coverage + # pytest-cov +cryptography==41.0.5 # via - # adal # azure-identity # azure-storage-blob # feast (setup.py) @@ -176,12 +147,10 @@ dask==2023.5.0 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7.post1 +debugpy==1.8.0 # via ipykernel decorator==5.1.1 - # via - # gcsfs - # ipython + # via ipython defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 @@ -207,17 +176,17 @@ exceptiongroup==1.1.3 # pytest execnet==2.0.2 # via pytest-xdist -executing==1.2.0 +executing==2.0.1 # via stack-data fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro -fastjsonschema==2.18.0 +fastjsonschema==2.19.0 # via nbformat -filelock==3.12.3 +filelock==3.13.1 # via # snowflake-connector-python # virtualenv @@ -229,22 +198,15 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.4.0 - # via - # aiohttp - # aiosignal -fsspec==2022.1.0 +fsspec==2023.9.2 # via - # adlfs # dask - # gcsfs -gcsfs==2022.1.0 - # via feast (setup.py) + # feast (setup.py) geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.1 +google-api-core[grpc]==2.14.0 # via # feast (setup.py) # firebase-admin @@ -256,24 +218,22 @@ google-api-core[grpc]==2.11.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.98.0 +google-api-python-client==2.108.0 # via firebase-admin -google-auth==2.22.0 +google-auth==2.23.4 # via - # gcsfs # google-api-core # google-api-python-client # google-auth-httplib2 - # google-auth-oauthlib # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.0 +google-auth-httplib2==0.1.1 # via google-api-python-client -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-bigquery[pandas]==3.11.4 - # via feast (setup.py) +google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) google-cloud-bigtable==2.21.0 @@ -287,20 +247,21 @@ google-cloud-core==2.3.3 # google-cloud-storage google-cloud-datastore==2.18.0 # via feast (setup.py) -google-cloud-firestore==2.11.1 +google-cloud-firestore==2.13.1 # via firebase-admin -google-cloud-storage==2.10.0 +google-cloud-storage==2.13.0 # via # feast (setup.py) # firebase-admin - # gcsfs google-crc32c==1.5.0 - # via google-resumable-media + # via + # google-cloud-storage + # google-resumable-media google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.60.0 +googleapis-common-protos[grpc]==1.61.0 # via # feast (setup.py) # google-api-core @@ -308,9 +269,11 @@ googleapis-common-protos[grpc]==1.60.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -grpc-google-iam-v1==0.12.6 +greenlet==3.0.1 + # via sqlalchemy +grpc-google-iam-v1==0.12.7 # via google-cloud-bigtable -grpcio==1.57.0 +grpcio==1.59.2 # via # feast (setup.py) # google-api-core @@ -322,15 +285,15 @@ grpcio==1.57.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-status==1.57.0 +grpcio-status==1.59.2 # via google-api-core -grpcio-testing==1.57.0 +grpcio-testing==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -344,17 +307,17 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.3 +httpcore==1.0.2 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) -identify==2.5.27 +identify==2.5.31 # via pre-commit idna==3.4 # via @@ -363,13 +326,13 @@ idna==3.4 # jsonschema # requests # snowflake-connector-python - # yarl imagesize==1.4.1 # via sphinx importlib-metadata==6.8.0 # via # build # dask + # feast (setup.py) # great-expectations # jupyter-client # jupyter-lsp @@ -377,31 +340,30 @@ importlib-metadata==6.8.0 # jupyterlab-server # nbconvert # sphinx -importlib-resources==6.0.1 +importlib-resources==6.1.1 # via + # feast (setup.py) # jsonschema # jsonschema-specifications # jupyterlab iniconfig==2.0.0 # via pytest -ipykernel==6.25.2 +ipykernel==6.26.0 # via jupyterlab -ipython==8.12.2 +ipython==8.12.3 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.0 +ipywidgets==8.1.1 # via great-expectations isodate==0.6.1 - # via - # azure-storage-blob - # msrest + # via azure-storage-blob isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.19.0 +jedi==0.19.1 # via ipython jinja2==3.1.2 # via @@ -426,7 +388,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.19.0 +jsonschema[format-nongpl]==4.20.0 # via # altair # feast (setup.py) @@ -434,14 +396,14 @@ jsonschema[format-nongpl]==4.19.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema -jupyter-client==8.3.1 +jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.3.1 +jupyter-core==5.5.0 # via # ipykernel # jupyter-client @@ -450,11 +412,11 @@ jupyter-core==5.3.1 # nbclient # nbconvert # nbformat -jupyter-events==0.7.0 +jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.0 # via jupyterlab -jupyter-server==2.7.3 +jupyter-server==2.10.1 # via # jupyter-lsp # jupyterlab @@ -463,21 +425,21 @@ jupyter-server==2.7.3 # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server -jupyterlab==4.0.5 +jupyterlab==4.0.8 # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-server==2.24.0 +jupyterlab-server==2.25.1 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.8 +jupyterlab-widgets==3.0.9 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd -makefun==1.15.1 +makefun==1.15.2 # via great-expectations markupsafe==2.1.3 # via @@ -494,7 +456,7 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==3.0.1 +mistune==3.0.2 # via # great-expectations # nbconvert @@ -504,25 +466,16 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.2 +moto==4.2.9 # via feast (setup.py) -msal==1.23.0 +msal==1.25.0 # via - # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.0.0 # via azure-identity -msgpack==1.0.5 +msgpack==1.0.7 # via cachecontrol -msrest==0.7.1 - # via msrestazure -msrestazure==0.6.4 - # via adlfs -multidict==6.0.4 - # via - # aiohttp - # yarl multiprocess==0.70.15 # via bytewax mypy==0.982 @@ -533,13 +486,13 @@ mypy-extensions==1.0.0 # via # black # mypy -mypy-protobuf==3.1 +mypy-protobuf==3.1.0 # via feast (setup.py) mysqlclient==2.2.0 # via feast (setup.py) -nbclient==0.8.0 +nbclient==0.9.0 # via nbconvert -nbconvert==7.8.0 +nbconvert==7.11.0 # via jupyter-server nbformat==5.9.2 # via @@ -547,11 +500,11 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.7 +nest-asyncio==1.5.8 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.3 +notebook==7.0.6 # via great-expectations notebook-shim==0.2.3 # via @@ -569,11 +522,9 @@ numpy==1.24.4 # scipy oauthlib==3.2.2 # via requests-oauthlib -oscrypto==1.3.0 - # via snowflake-connector-python overrides==7.4.0 # via jupyter-server -packaging==23.1 +packaging==23.2 # via # build # dask @@ -607,11 +558,11 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -partd==1.4.0 +partd==1.4.1 # via dask pathspec==0.11.2 # via black -pbr==5.11.1 +pbr==6.0.0 # via mock pexpect==4.8.0 # via ipython @@ -621,7 +572,7 @@ pip-tools==7.3.0 # via feast (setup.py) pkgutil-resolve-name==1.3.10 # via jsonschema -platformdirs==3.8.1 +platformdirs==3.11.0 # via # black # jupyter-core @@ -631,13 +582,13 @@ pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.7.0 +portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via jupyter-server -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython proto-plus==1.22.3 # via @@ -669,7 +620,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -683,7 +634,7 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==10.0.1 +pyarrow==14.0.1 # via # db-dtypes # feast (setup.py) @@ -701,9 +652,7 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.18.0 - # via snowflake-connector-python -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -718,16 +667,15 @@ pygments==2.16.1 # sphinx pyjwt[crypto]==2.8.0 # via - # adal # msal # snowflake-connector-python -pymssql==2.2.8 +pymssql==2.2.10 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==4.0.39 +pyodbc==5.0.1 # via feast (setup.py) -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via snowflake-connector-python pyparsing==3.1.1 # via @@ -735,9 +683,9 @@ pyparsing==3.1.1 # httplib2 pyproject-hooks==1.0.0 # via build -pyspark==3.4.1 +pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.1 +pytest==7.4.3 # via # feast (setup.py) # pytest-benchmark @@ -759,11 +707,10 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.3.1 +pytest-xdist==3.4.0 # via feast (setup.py) python-dateutil==2.8.2 # via - # adal # arrow # botocore # google-cloud-bigquery @@ -801,23 +748,19 @@ pyzmq==25.1.1 # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.8.8 +regex==2023.10.3 # via feast (setup.py) requests==2.31.0 # via - # adal - # adlfs # azure-core - # azure-datalake-store # cachecontrol # docker # feast (setup.py) - # gcsfs # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -826,18 +769,14 @@ requests==2.31.0 # kubernetes # moto # msal - # msrest # requests-oauthlib # responses # snowflake-connector-python # sphinx # trino requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes - # msrest -responses==0.23.3 + # via kubernetes +responses==0.24.1 # via moto rfc3339-validator==0.1.4 # via @@ -849,7 +788,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -857,9 +796,9 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.6.2 +s3transfer==0.7.0 # via boto3 scipy==1.10.1 # via great-expectations @@ -872,13 +811,10 @@ six==1.16.0 # bleach # cassandra-driver # geomet - # google-auth - # google-auth-httplib2 # happybase # isodate # kubernetes # mock - # msrestazure # pandavro # python-dateutil # rfc3339-validator @@ -886,12 +822,13 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.1.1 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.5.0 + # via + # feast (setup.py) + # snowflake-connector-python sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -910,11 +847,13 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy -stack-data==0.6.2 +stack-data==0.6.3 # via ipython starlette==0.27.0 # via fastapi @@ -922,13 +861,13 @@ tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 # via feast (setup.py) -terminado==0.17.1 +terminado==0.18.0 # via # jupyter-server # jupyter-server-terminals testcontainers==3.7.1 # via feast (setup.py) -thriftpy2==0.4.16 +thriftpy2==0.4.17 # via happybase tinycss2==1.2.1 # via nbconvert @@ -944,7 +883,7 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest -tomlkit==0.12.1 +tomlkit==0.12.3 # via snowflake-connector-python toolz==0.12.0 # via @@ -963,7 +902,7 @@ tqdm==4.66.1 # via # feast (setup.py) # great-expectations -traitlets==5.9.0 +traitlets==5.13.0 # via # comm # ipykernel @@ -978,7 +917,7 @@ traitlets==5.9.0 # nbclient # nbconvert # nbformat -trino==0.326.0 +trino==0.327.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -988,34 +927,33 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.2 +types-pyopenssl==23.3.0.0 # via types-redis types-python-dateutil==2.8.19.14 - # via feast (setup.py) -types-pytz==2023.3.0.1 - # via feast (setup.py) -types-pyyaml==6.0.12.11 # via + # arrow # feast (setup.py) - # responses -types-redis==4.6.0.5 +types-pytz==2023.3.1.1 + # via feast (setup.py) +types-pyyaml==6.0.12.12 # via feast (setup.py) -types-requests==2.31.0.2 +types-redis==4.6.0.10 # via feast (setup.py) -types-setuptools==68.2.0.0 +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==68.2.0.1 # via feast (setup.py) types-tabulate==0.9.0.3 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # async-lru # azure-core # azure-storage-blob # black # fastapi - # filelock # great-expectations # ipython # mypy @@ -1024,7 +962,7 @@ typing-extensions==4.7.1 # sqlalchemy2-stubs # starlette # uvicorn -tzlocal==5.0.1 +tzlocal==5.2 # via # great-expectations # trino @@ -1032,12 +970,11 @@ uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # docker # feast (setup.py) - # google-auth # great-expectations # kubernetes # minio @@ -1045,9 +982,11 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 # via @@ -1055,9 +994,9 @@ virtualenv==20.23.0 # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.6 +wcwidth==0.2.10 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -1065,26 +1004,24 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.2 +websocket-client==1.6.4 # via # docker # jupyter-server # kubernetes -websockets==11.0.3 +websockets==12.0 # via uvicorn -werkzeug==2.3.7 +werkzeug==3.0.1 # via moto -wheel==0.41.2 +wheel==0.41.3 # via pip-tools -widgetsnbextension==4.0.8 +widgetsnbextension==4.0.9 # via ipywidgets -wrapt==1.15.0 +wrapt==1.16.0 # via testcontainers xmltodict==0.13.0 # via moto -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 +zipp==3.17.0 # via # importlib-metadata # importlib-resources diff --git a/sdk/python/requirements/py3.8-requirements.txt b/sdk/python/requirements/py3.8-requirements.txt index 636f886133..3f6a15a1ef 100644 --- a/sdk/python/requirements/py3.8-requirements.txt +++ b/sdk/python/requirements/py3.8-requirements.txt @@ -6,7 +6,7 @@ # anyio==4.0.0 # via - # httpcore + # httpx # starlette # watchfiles appdirs==1.4.4 @@ -23,7 +23,7 @@ certifi==2023.7.22 # httpcore # httpx # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -32,7 +32,7 @@ click==8.1.7 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) @@ -44,25 +44,27 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.9.0 +fsspec==2023.10.0 # via dask -grpcio==1.57.0 +greenlet==3.0.1 + # via sqlalchemy +grpcio==1.59.2 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -70,11 +72,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.3 +httpcore==1.0.2 # via httpx -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) idna==3.4 # via @@ -82,16 +84,19 @@ idna==3.4 # httpx # requests importlib-metadata==6.8.0 - # via dask -importlib-resources==6.0.1 # via + # dask + # feast (setup.py) +importlib-resources==6.1.1 + # via + # feast (setup.py) # jsonschema # jsonschema-specifications jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.19.0 +jsonschema==4.20.0 # via feast (setup.py) -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema locket==1.0.0 # via partd @@ -101,11 +106,11 @@ mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.5.1 +mypy==1.7.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy -mypy-protobuf==3.1 +mypy-protobuf==3.1.0 # via feast (setup.py) numpy==1.24.4 # via @@ -113,7 +118,7 @@ numpy==1.24.4 # pandas # pandavro # pyarrow -packaging==23.1 +packaging==23.2 # via # dask # gunicorn @@ -123,7 +128,7 @@ pandas==1.5.3 # pandavro pandavro==1.5.2 # via feast (setup.py) -partd==1.4.0 +partd==1.4.1 # via dask pkgutil-resolve-name==1.3.10 # via jsonschema @@ -137,9 +142,9 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==11.0.0 +pyarrow==14.0.1 # via feast (setup.py) -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -156,13 +161,13 @@ pyyaml==6.0.1 # dask # feast (setup.py) # uvicorn -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -173,11 +178,12 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy starlette==0.27.0 # via fastapi @@ -197,9 +203,9 @@ tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.1 +types-protobuf==4.24.0.4 # via mypy-protobuf -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # fastapi # mypy @@ -207,19 +213,21 @@ typing-extensions==4.7.1 # sqlalchemy2-stubs # starlette # uvicorn -urllib3==2.0.4 +urllib3==2.1.0 # via requests -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -websockets==11.0.3 +websockets==12.0 # via uvicorn -zipp==3.16.2 +zipp==3.17.0 # via # importlib-metadata # importlib-resources diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index ad19f9e8bd..5bc9210fd8 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,93 +4,64 @@ # # pip-compile --extra=ci --output-file=sdk/python/requirements/py3.9-ci-requirements.txt # -adal==1.2.7 - # via msrestazure -adlfs==0.5.9 - # via feast (setup.py) -aiohttp==3.8.5 - # via - # adlfs - # gcsfs -aiosignal==1.3.1 - # via aiohttp alabaster==0.7.13 # via sphinx altair==4.2.0 # via great-expectations anyio==4.0.0 # via - # httpcore + # httpx # jupyter-server # starlette # watchfiles appdirs==1.4.4 # via fissix -appnope==0.1.3 - # via - # ipykernel - # ipython argon2-cffi==23.1.0 # via jupyter-server argon2-cffi-bindings==21.2.0 # via argon2-cffi -arrow==1.2.3 +arrow==1.3.0 # via isoduration asn1crypto==1.5.1 - # via - # oscrypto - # snowflake-connector-python + # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.0 +asttokens==2.4.1 # via stack-data async-lru==2.0.4 # via jupyterlab async-timeout==4.0.3 - # via - # aiohttp - # redis + # via redis attrs==23.1.0 # via - # aiohttp # bowler # jsonschema # referencing avro==1.10.0 # via feast (setup.py) -azure-core==1.29.3 +azure-core==1.29.5 # via - # adlfs # azure-identity # azure-storage-blob - # msrest -azure-datalake-store==0.0.53 - # via adlfs -azure-identity==1.14.0 - # via - # adlfs - # feast (setup.py) -azure-storage-blob==12.17.0 - # via - # adlfs - # feast (setup.py) -babel==2.12.1 +azure-identity==1.15.0 + # via feast (setup.py) +azure-storage-blob==12.19.0 + # via feast (setup.py) +babel==2.13.1 # via # jupyterlab-server # sphinx -backcall==0.2.0 - # via ipython beautifulsoup4==4.12.2 # via nbconvert black==22.12.0 # via feast (setup.py) -bleach==6.0.0 +bleach==6.1.0 # via nbconvert -boto3==1.28.42 +boto3==1.29.2 # via # feast (setup.py) # moto -botocore==1.31.42 +botocore==1.32.2 # via # boto3 # moto @@ -105,7 +76,7 @@ bytewax==0.15.1 # via feast (setup.py) cachecontrol==0.13.1 # via firebase-admin -cachetools==5.3.1 +cachetools==5.3.2 # via google-auth cassandra-driver==3.28.0 # via feast (setup.py) @@ -115,20 +86,17 @@ certifi==2023.7.22 # httpx # kubernetes # minio - # msrest # requests # snowflake-connector-python -cffi==1.15.1 +cffi==1.16.0 # via # argon2-cffi-bindings - # azure-datalake-store # cryptography # snowflake-connector-python cfgv==3.4.0 # via pre-commit -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via - # aiohttp # requests # snowflake-connector-python click==8.1.7 @@ -142,21 +110,22 @@ click==8.1.7 # moreorless # pip-tools # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via # feast (setup.py) # great-expectations -comm==0.1.4 +comm==0.2.0 # via # ipykernel # ipywidgets -coverage[toml]==7.3.1 - # via pytest-cov -cryptography==41.0.3 +coverage[toml]==7.3.2 + # via + # coverage + # pytest-cov +cryptography==41.0.5 # via - # adal # azure-identity # azure-storage-blob # feast (setup.py) @@ -168,16 +137,14 @@ cryptography==41.0.3 # snowflake-connector-python # types-pyopenssl # types-redis -dask==2023.9.1 +dask==2023.11.0 # via feast (setup.py) db-dtypes==1.1.1 # via google-cloud-bigquery -debugpy==1.6.7.post1 +debugpy==1.8.0 # via ipykernel decorator==5.1.1 - # via - # gcsfs - # ipython + # via ipython defusedxml==0.7.1 # via nbconvert deprecation==2.1.0 @@ -204,17 +171,17 @@ exceptiongroup==1.1.3 # pytest execnet==2.0.2 # via pytest-xdist -executing==1.2.0 +executing==2.0.1 # via stack-data fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro -fastjsonschema==2.18.0 +fastjsonschema==2.19.0 # via nbformat -filelock==3.12.3 +filelock==3.13.1 # via # snowflake-connector-python # virtualenv @@ -226,22 +193,15 @@ flake8==6.0.0 # via feast (setup.py) fqdn==1.5.1 # via jsonschema -frozenlist==1.4.0 +fsspec==2023.9.2 # via - # aiohttp - # aiosignal -fsspec==2022.1.0 - # via - # adlfs # dask - # gcsfs -gcsfs==2022.1.0 - # via feast (setup.py) + # feast (setup.py) geojson==2.5.0 # via rockset geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.11.1 +google-api-core[grpc]==2.14.0 # via # feast (setup.py) # firebase-admin @@ -253,24 +213,22 @@ google-api-core[grpc]==2.11.1 # google-cloud-datastore # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.98.0 +google-api-python-client==2.108.0 # via firebase-admin -google-auth==2.22.0 +google-auth==2.23.4 # via - # gcsfs # google-api-core # google-api-python-client # google-auth-httplib2 - # google-auth-oauthlib # google-cloud-core # google-cloud-storage # kubernetes -google-auth-httplib2==0.1.0 +google-auth-httplib2==0.1.1 # via google-api-python-client -google-auth-oauthlib==1.0.0 - # via gcsfs -google-cloud-bigquery[pandas]==3.11.4 - # via feast (setup.py) +google-cloud-bigquery[pandas]==3.12.0 + # via + # feast (setup.py) + # google-cloud-bigquery google-cloud-bigquery-storage==2.22.0 # via feast (setup.py) google-cloud-bigtable==2.21.0 @@ -284,20 +242,21 @@ google-cloud-core==2.3.3 # google-cloud-storage google-cloud-datastore==2.18.0 # via feast (setup.py) -google-cloud-firestore==2.11.1 +google-cloud-firestore==2.13.1 # via firebase-admin -google-cloud-storage==2.10.0 +google-cloud-storage==2.13.0 # via # feast (setup.py) # firebase-admin - # gcsfs google-crc32c==1.5.0 - # via google-resumable-media + # via + # google-cloud-storage + # google-resumable-media google-resumable-media==2.6.0 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.60.0 +googleapis-common-protos[grpc]==1.61.0 # via # feast (setup.py) # google-api-core @@ -305,9 +264,11 @@ googleapis-common-protos[grpc]==1.60.0 # grpcio-status great-expectations==0.15.50 # via feast (setup.py) -grpc-google-iam-v1==0.12.6 +greenlet==3.0.1 + # via sqlalchemy +grpc-google-iam-v1==0.12.7 # via google-cloud-bigtable -grpcio==1.57.0 +grpcio==1.59.2 # via # feast (setup.py) # google-api-core @@ -319,15 +280,15 @@ grpcio==1.57.0 # grpcio-status # grpcio-testing # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-status==1.57.0 +grpcio-status==1.59.2 # via google-api-core -grpcio-testing==1.57.0 +grpcio-testing==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -341,17 +302,17 @@ hazelcast-python-client==5.3.0 # via feast (setup.py) hiredis==2.2.3 # via feast (setup.py) -httpcore==0.17.3 +httpcore==1.0.2 # via httpx httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) -identify==2.5.27 +identify==2.5.31 # via pre-commit idna==3.4 # via @@ -360,13 +321,13 @@ idna==3.4 # jsonschema # requests # snowflake-connector-python - # yarl imagesize==1.4.1 # via sphinx importlib-metadata==6.8.0 # via # build # dask + # feast (setup.py) # great-expectations # jupyter-client # jupyter-lsp @@ -374,26 +335,26 @@ importlib-metadata==6.8.0 # jupyterlab-server # nbconvert # sphinx +importlib-resources==6.1.1 + # via feast (setup.py) iniconfig==2.0.0 # via pytest -ipykernel==6.25.2 +ipykernel==6.26.0 # via jupyterlab -ipython==8.15.0 +ipython==8.17.2 # via # great-expectations # ipykernel # ipywidgets -ipywidgets==8.1.0 +ipywidgets==8.1.1 # via great-expectations isodate==0.6.1 - # via - # azure-storage-blob - # msrest + # via azure-storage-blob isoduration==20.11.0 # via jsonschema isort==5.12.0 # via feast (setup.py) -jedi==0.19.0 +jedi==0.19.1 # via ipython jinja2==3.1.2 # via @@ -418,7 +379,7 @@ jsonpointer==2.4 # via # jsonpatch # jsonschema -jsonschema[format-nongpl]==4.19.0 +jsonschema[format-nongpl]==4.20.0 # via # altair # feast (setup.py) @@ -426,14 +387,14 @@ jsonschema[format-nongpl]==4.19.0 # jupyter-events # jupyterlab-server # nbformat -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema -jupyter-client==8.3.1 +jupyter-client==8.6.0 # via # ipykernel # jupyter-server # nbclient -jupyter-core==5.3.1 +jupyter-core==5.5.0 # via # ipykernel # jupyter-client @@ -442,11 +403,11 @@ jupyter-core==5.3.1 # nbclient # nbconvert # nbformat -jupyter-events==0.7.0 +jupyter-events==0.9.0 # via jupyter-server jupyter-lsp==2.2.0 # via jupyterlab -jupyter-server==2.7.3 +jupyter-server==2.10.1 # via # jupyter-lsp # jupyterlab @@ -455,21 +416,21 @@ jupyter-server==2.7.3 # notebook-shim jupyter-server-terminals==0.4.4 # via jupyter-server -jupyterlab==4.0.5 +jupyterlab==4.0.8 # via notebook jupyterlab-pygments==0.2.2 # via nbconvert -jupyterlab-server==2.24.0 +jupyterlab-server==2.25.1 # via # jupyterlab # notebook -jupyterlab-widgets==3.0.8 +jupyterlab-widgets==3.0.9 # via ipywidgets kubernetes==20.13.0 # via feast (setup.py) locket==1.0.0 # via partd -makefun==1.15.1 +makefun==1.15.2 # via great-expectations markupsafe==2.1.3 # via @@ -486,7 +447,7 @@ mccabe==0.7.0 # via flake8 minio==7.1.0 # via feast (setup.py) -mistune==3.0.1 +mistune==3.0.2 # via # great-expectations # nbconvert @@ -496,25 +457,16 @@ mock==2.0.0 # via feast (setup.py) moreorless==0.4.0 # via bowler -moto==4.2.2 +moto==4.2.9 # via feast (setup.py) -msal==1.23.0 +msal==1.25.0 # via - # azure-datalake-store # azure-identity # msal-extensions msal-extensions==1.0.0 # via azure-identity -msgpack==1.0.5 +msgpack==1.0.7 # via cachecontrol -msrest==0.7.1 - # via msrestazure -msrestazure==0.6.4 - # via adlfs -multidict==6.0.4 - # via - # aiohttp - # yarl multiprocess==0.70.15 # via bytewax mypy==0.982 @@ -529,9 +481,9 @@ mypy-protobuf==3.1.0 # via feast (setup.py) mysqlclient==2.2.0 # via feast (setup.py) -nbclient==0.8.0 +nbclient==0.9.0 # via nbconvert -nbconvert==7.8.0 +nbconvert==7.11.0 # via jupyter-server nbformat==5.9.2 # via @@ -539,17 +491,17 @@ nbformat==5.9.2 # jupyter-server # nbclient # nbconvert -nest-asyncio==1.5.7 +nest-asyncio==1.5.8 # via ipykernel nodeenv==1.8.0 # via pre-commit -notebook==7.0.3 +notebook==7.0.6 # via great-expectations notebook-shim==0.2.3 # via # jupyterlab # notebook -numpy==1.25.2 +numpy==1.24.4 # via # altair # db-dtypes @@ -561,11 +513,9 @@ numpy==1.25.2 # scipy oauthlib==3.2.2 # via requests-oauthlib -oscrypto==1.3.0 - # via snowflake-connector-python overrides==7.4.0 # via jupyter-server -packaging==23.1 +packaging==23.2 # via # build # dask @@ -599,19 +549,17 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -partd==1.4.0 +partd==1.4.1 # via dask pathspec==0.11.2 # via black -pbr==5.11.1 +pbr==6.0.0 # via mock pexpect==4.8.0 # via ipython -pickleshare==0.7.5 - # via ipython pip-tools==7.3.0 # via feast (setup.py) -platformdirs==3.8.1 +platformdirs==3.11.0 # via # black # jupyter-core @@ -621,13 +569,13 @@ pluggy==1.3.0 # via pytest ply==3.11 # via thriftpy2 -portalocker==2.7.0 +portalocker==2.8.2 # via msal-extensions pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.17.1 +prometheus-client==0.18.0 # via jupyter-server -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via ipython proto-plus==1.22.3 # via @@ -659,7 +607,7 @@ psutil==5.9.0 # via # feast (setup.py) # ipykernel -psycopg2-binary==2.9.7 +psycopg2-binary==2.9.9 # via feast (setup.py) ptyprocess==0.7.0 # via @@ -673,7 +621,7 @@ py-cpuinfo==9.0.0 # via pytest-benchmark py4j==0.10.9.7 # via pyspark -pyarrow==10.0.1 +pyarrow==14.0.1 # via # db-dtypes # feast (setup.py) @@ -691,9 +639,7 @@ pycodestyle==2.10.0 # via flake8 pycparser==2.21 # via cffi -pycryptodomex==3.18.0 - # via snowflake-connector-python -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -708,16 +654,15 @@ pygments==2.16.1 # sphinx pyjwt[crypto]==2.8.0 # via - # adal # msal # snowflake-connector-python -pymssql==2.2.8 +pymssql==2.2.10 # via feast (setup.py) pymysql==1.1.0 # via feast (setup.py) -pyodbc==4.0.39 +pyodbc==5.0.1 # via feast (setup.py) -pyopenssl==23.2.0 +pyopenssl==23.3.0 # via snowflake-connector-python pyparsing==3.1.1 # via @@ -725,9 +670,9 @@ pyparsing==3.1.1 # httplib2 pyproject-hooks==1.0.0 # via build -pyspark==3.4.1 +pyspark==3.5.0 # via feast (setup.py) -pytest==7.4.1 +pytest==7.4.3 # via # feast (setup.py) # pytest-benchmark @@ -749,11 +694,10 @@ pytest-ordering==0.6 # via feast (setup.py) pytest-timeout==1.4.2 # via feast (setup.py) -pytest-xdist==3.3.1 +pytest-xdist==3.4.0 # via feast (setup.py) python-dateutil==2.8.2 # via - # adal # arrow # botocore # google-cloud-bigquery @@ -790,23 +734,19 @@ pyzmq==25.1.1 # jupyter-server redis==4.6.0 # via feast (setup.py) -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications # jupyter-events -regex==2023.8.8 +regex==2023.10.3 # via feast (setup.py) requests==2.31.0 # via - # adal - # adlfs # azure-core - # azure-datalake-store # cachecontrol # docker # feast (setup.py) - # gcsfs # google-api-core # google-cloud-bigquery # google-cloud-storage @@ -815,18 +755,14 @@ requests==2.31.0 # kubernetes # moto # msal - # msrest # requests-oauthlib # responses # snowflake-connector-python # sphinx # trino requests-oauthlib==1.3.1 - # via - # google-auth-oauthlib - # kubernetes - # msrest -responses==0.23.3 + # via kubernetes +responses==0.24.1 # via moto rfc3339-validator==0.1.4 # via @@ -838,7 +774,7 @@ rfc3986-validator==0.1.1 # jupyter-events rockset==2.1.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -846,11 +782,11 @@ rsa==4.9 # via google-auth ruamel-yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.7 +ruamel-yaml-clib==0.2.8 # via ruamel-yaml -s3transfer==0.6.2 +s3transfer==0.7.0 # via boto3 -scipy==1.11.2 +scipy==1.11.3 # via great-expectations send2trash==1.8.2 # via jupyter-server @@ -861,13 +797,10 @@ six==1.16.0 # bleach # cassandra-driver # geomet - # google-auth - # google-auth-httplib2 # happybase # isodate # kubernetes # mock - # msrestazure # pandavro # python-dateutil # rfc3339-validator @@ -875,12 +808,13 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.1.1 - # via feast (setup.py) +snowflake-connector-python[pandas]==3.5.0 + # via + # feast (setup.py) + # snowflake-connector-python sortedcontainers==2.4.0 # via snowflake-connector-python soupsieve==2.5 @@ -905,11 +839,13 @@ sphinxcontrib-qthelp==1.0.6 # via sphinx sphinxcontrib-serializinghtml==1.1.9 # via sphinx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy -stack-data==0.6.2 +stack-data==0.6.3 # via ipython starlette==0.27.0 # via fastapi @@ -917,13 +853,13 @@ tabulate==0.9.0 # via feast (setup.py) tenacity==8.2.3 # via feast (setup.py) -terminado==0.17.1 +terminado==0.18.0 # via # jupyter-server # jupyter-server-terminals testcontainers==3.7.1 # via feast (setup.py) -thriftpy2==0.4.16 +thriftpy2==0.4.17 # via happybase tinycss2==1.2.1 # via nbconvert @@ -939,7 +875,7 @@ tomli==2.0.1 # pip-tools # pyproject-hooks # pytest -tomlkit==0.12.1 +tomlkit==0.12.3 # via snowflake-connector-python toolz==0.12.0 # via @@ -958,7 +894,7 @@ tqdm==4.66.1 # via # feast (setup.py) # great-expectations -traitlets==5.9.0 +traitlets==5.13.0 # via # comm # ipykernel @@ -973,7 +909,7 @@ traitlets==5.9.0 # nbclient # nbconvert # nbformat -trino==0.326.0 +trino==0.327.0 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) @@ -983,34 +919,33 @@ types-protobuf==3.19.22 # mypy-protobuf types-pymysql==1.1.0.1 # via feast (setup.py) -types-pyopenssl==23.2.0.2 +types-pyopenssl==23.3.0.0 # via types-redis types-python-dateutil==2.8.19.14 - # via feast (setup.py) -types-pytz==2023.3.0.1 - # via feast (setup.py) -types-pyyaml==6.0.12.11 # via + # arrow # feast (setup.py) - # responses -types-redis==4.6.0.5 +types-pytz==2023.3.1.1 + # via feast (setup.py) +types-pyyaml==6.0.12.12 # via feast (setup.py) -types-requests==2.31.0.2 +types-redis==4.6.0.10 # via feast (setup.py) -types-setuptools==68.2.0.0 +types-requests==2.30.0.0 + # via feast (setup.py) +types-setuptools==68.2.0.1 # via feast (setup.py) types-tabulate==0.9.0.3 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # async-lru # azure-core # azure-storage-blob # black # fastapi - # filelock # great-expectations # ipython # mypy @@ -1019,7 +954,7 @@ typing-extensions==4.7.1 # sqlalchemy2-stubs # starlette # uvicorn -tzlocal==5.0.1 +tzlocal==5.2 # via # great-expectations # trino @@ -1027,12 +962,11 @@ uri-template==1.3.0 # via jsonschema uritemplate==4.1.1 # via google-api-python-client -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore # docker # feast (setup.py) - # google-auth # great-expectations # kubernetes # minio @@ -1040,9 +974,11 @@ urllib3==1.26.16 # responses # rockset # snowflake-connector-python -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn virtualenv==20.23.0 # via @@ -1050,9 +986,9 @@ virtualenv==20.23.0 # pre-commit volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.6 +wcwidth==0.2.10 # via prompt-toolkit webcolors==1.13 # via jsonschema @@ -1060,27 +996,27 @@ webencodings==0.5.1 # via # bleach # tinycss2 -websocket-client==1.6.2 +websocket-client==1.6.4 # via # docker # jupyter-server # kubernetes -websockets==11.0.3 +websockets==12.0 # via uvicorn -werkzeug==2.3.7 +werkzeug==3.0.1 # via moto -wheel==0.41.2 +wheel==0.41.3 # via pip-tools -widgetsnbextension==4.0.8 +widgetsnbextension==4.0.9 # via ipywidgets -wrapt==1.15.0 +wrapt==1.16.0 # via testcontainers xmltodict==0.13.0 # via moto -yarl==1.9.2 - # via aiohttp -zipp==3.16.2 - # via importlib-metadata +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 7d30fd3452..120ecf9eb3 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -6,7 +6,7 @@ # anyio==4.0.0 # via - # httpcore + # httpx # starlette # watchfiles appdirs==1.4.4 @@ -23,7 +23,7 @@ certifi==2023.7.22 # httpcore # httpx # requests -charset-normalizer==3.2.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -32,11 +32,11 @@ click==8.1.7 # feast (setup.py) # moreorless # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask==2023.9.1 +dask==2023.11.0 # via feast (setup.py) dill==0.3.7 # via feast (setup.py) @@ -44,25 +44,27 @@ exceptiongroup==1.1.3 # via anyio fastapi==0.99.1 # via feast (setup.py) -fastavro==1.8.2 +fastavro==1.9.0 # via # feast (setup.py) # pandavro fissix==21.11.13 # via bowler -fsspec==2023.9.0 +fsspec==2023.10.0 # via dask -grpcio==1.57.0 +greenlet==3.0.1 + # via sqlalchemy +grpcio==1.59.2 # via # feast (setup.py) # grpcio-health-checking # grpcio-reflection # grpcio-tools -grpcio-health-checking==1.57.0 +grpcio-health-checking==1.59.2 # via feast (setup.py) -grpcio-reflection==1.57.0 +grpcio-reflection==1.59.2 # via feast (setup.py) -grpcio-tools==1.57.0 +grpcio-tools==1.59.2 # via feast (setup.py) gunicorn==21.2.0 # via feast (setup.py) @@ -70,11 +72,11 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==0.17.3 +httpcore==1.0.2 # via httpx -httptools==0.6.0 +httptools==0.6.1 # via uvicorn -httpx==0.24.1 +httpx==0.25.1 # via feast (setup.py) idna==3.4 # via @@ -82,12 +84,16 @@ idna==3.4 # httpx # requests importlib-metadata==6.8.0 - # via dask + # via + # dask + # feast (setup.py) +importlib-resources==6.1.1 + # via feast (setup.py) jinja2==3.1.2 # via feast (setup.py) -jsonschema==4.19.0 +jsonschema==4.20.0 # via feast (setup.py) -jsonschema-specifications==2023.7.1 +jsonschema-specifications==2023.11.1 # via jsonschema locket==1.0.0 # via partd @@ -97,19 +103,19 @@ mmh3==4.0.1 # via feast (setup.py) moreorless==0.4.0 # via bowler -mypy==1.5.1 +mypy==1.7.0 # via sqlalchemy mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.1.0 # via feast (setup.py) -numpy==1.25.2 +numpy==1.24.4 # via # feast (setup.py) # pandas # pandavro # pyarrow -packaging==23.1 +packaging==23.2 # via # dask # gunicorn @@ -119,7 +125,7 @@ pandas==1.5.3 # pandavro pandavro==1.5.2 # via feast (setup.py) -partd==1.4.0 +partd==1.4.1 # via dask proto-plus==1.22.3 # via feast (setup.py) @@ -131,9 +137,9 @@ protobuf==4.23.3 # grpcio-tools # mypy-protobuf # proto-plus -pyarrow==11.0.0 +pyarrow==14.0.1 # via feast (setup.py) -pydantic==1.10.12 +pydantic==1.10.13 # via # fastapi # feast (setup.py) @@ -150,13 +156,13 @@ pyyaml==6.0.1 # dask # feast (setup.py) # uvicorn -referencing==0.30.2 +referencing==0.31.0 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via feast (setup.py) -rpds-py==0.10.2 +rpds-py==0.13.0 # via # jsonschema # referencing @@ -167,11 +173,12 @@ six==1.16.0 sniffio==1.3.0 # via # anyio - # httpcore # httpx -sqlalchemy[mypy]==1.4.49 - # via feast (setup.py) -sqlalchemy2-stubs==0.0.2a35 +sqlalchemy[mypy]==1.4.50 + # via + # feast (setup.py) + # sqlalchemy +sqlalchemy2-stubs==0.0.2a37 # via sqlalchemy starlette==0.27.0 # via fastapi @@ -191,9 +198,9 @@ tqdm==4.66.1 # via feast (setup.py) typeguard==2.13.3 # via feast (setup.py) -types-protobuf==4.24.0.1 +types-protobuf==4.24.0.4 # via mypy-protobuf -typing-extensions==4.7.1 +typing-extensions==4.8.0 # via # fastapi # mypy @@ -201,20 +208,24 @@ typing-extensions==4.7.1 # sqlalchemy2-stubs # starlette # uvicorn -urllib3==2.0.4 +urllib3==2.1.0 # via requests -uvicorn[standard]==0.23.2 - # via feast (setup.py) -uvloop==0.17.0 +uvicorn[standard]==0.24.0.post1 + # via + # feast (setup.py) + # uvicorn +uvloop==0.19.0 # via uvicorn volatile==2.1.0 # via bowler -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -websockets==11.0.3 +websockets==12.0 # via uvicorn -zipp==3.16.2 - # via importlib-metadata +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/setup.py b/setup.py index 573ab54d51..33bf76e181 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ from setuptools.command.install import install except ImportError: - from distutils.command.build_ext import build_ext as _build_ext from distutils.command.build_py import build_py from distutils.core import setup @@ -54,14 +53,14 @@ "Jinja2>=2,<4", "jsonschema", "mmh3", - "numpy>=1.22,<3", + "numpy>=1.22,<1.25", "pandas>=1.4.3,<2", # For some reason pandavro higher than 1.5.* only support pandas less than 1.3. "pandavro~=1.5.0", # Higher than 4.23.4 seems to cause a seg fault "protobuf<4.23.4,>3.20", "proto-plus>=1.20.0,<2", - "pyarrow>=4,<12", + "pyarrow>=4", "pydantic>=1,<2", "pygments>=2.12.0,<3", "PyYAML>=5.4.0,<7", @@ -79,16 +78,19 @@ "bowler", # Needed for automatic repo upgrades # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). "httpx>=0.23.3", + "importlib-resources>=6.0.0,<7", + "importlib_metadata>=6.8.0,<7", ] GCP_REQUIRED = [ "google-api-core>=1.23.0,<3", "googleapis-common-protos>=1.52.0,<2", - "google-cloud-bigquery[pandas]>=2,<4", + "google-cloud-bigquery[pandas]>=2,<3.13.0", "google-cloud-bigquery-storage >= 2.0.0,<3", "google-cloud-datastore>=2.1.0,<3", "google-cloud-storage>=1.34.0,<3", "google-cloud-bigtable>=2.11.0,<3", + "fsspec<2023.10.0", ] REDIS_REQUIRED = [ @@ -96,7 +98,7 @@ "hiredis>=2.0.0,<3", ] -AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2"] +AWS_REQUIRED = ["boto3>=1.17.0,<2", "docker>=5.0.2", "fsspec<2023.10.0"] BYTEWAX_REQUIRED = ["bytewax==0.15.1", "docker>=5.0.2", "kubernetes<=20.13.0"] @@ -156,8 +158,8 @@ "moto", "mypy>=0.981,<0.990", "avro==1.10.0", - "gcsfs>=0.4.0,<=2022.01.0", - "urllib3>=1.25.4,<2", + "fsspec<2023.10.0", + "urllib3>=1.25.4,<3", "psutil==5.9.0", "py>=1.11.0", # https://github.com/pytest-dev/pytest/issues/10420 "pytest>=6.0.0,<8", @@ -170,7 +172,6 @@ "pytest-mock==1.10.4", "Sphinx>4.0.0,<7", "testcontainers>=3.5,<4", - "adlfs==0.5.9", "firebase-admin>=5.2.0,<6", "pre-commit<3.3.2", "assertpy==1.1", @@ -181,10 +182,10 @@ "types-pytz", "types-PyYAML", "types-redis", - "types-requests", + "types-requests<2.31.0", "types-setuptools", "types-tabulate", - "virtualenv<20.24.2" + "virtualenv<20.24.2", ] + GCP_REQUIRED + REDIS_REQUIRED diff --git a/ui/CONTRIBUTING.md b/ui/CONTRIBUTING.md index 970bd3676c..3c13759e26 100644 --- a/ui/CONTRIBUTING.md +++ b/ui/CONTRIBUTING.md @@ -91,7 +91,7 @@ The Feast UI is published as a module to NPM and can be found here: https://www. ### Requirements To publish a new version of the module, you will need: -- to be part of the @feast-dev team in NPM. Ask `#feast-development` on http://slack.feast.dev to add you if necessary. +- to be part of the @feast-dev team in NPM. - to [login to your NPM account on the command line](https://docs.npmjs.com/cli/v8/commands/npm-adduser). ### Steps for Publishing diff --git a/ui/package.json b/ui/package.json index dc16a1e7a6..c826737cf6 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,6 +1,6 @@ { "name": "@feast-dev/feast-ui", - "version": "0.34.0", + "version": "0.35.0", "private": false, "files": [ "dist" diff --git a/ui/yarn.lock b/ui/yarn.lock index 49bff13372..78fe6863bf 100644 --- a/ui/yarn.lock +++ b/ui/yarn.lock @@ -5708,9 +5708,9 @@ focus-lock@^0.11.2: tslib "^2.0.3" follow-redirects@^1.0.0: - version "1.14.7" - resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.14.7.tgz#2004c02eb9436eee9a21446a6477debf17e81685" - integrity sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ== + version "1.15.4" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf" + integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw== fork-ts-checker-webpack-plugin@^6.5.0: version "6.5.0"