diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml new file mode 100644 index 00000000000..6ee866781dc --- /dev/null +++ b/.github/workflows/build_wheels.yml @@ -0,0 +1,210 @@ +name: build_wheels + +# Call this workflow from other workflows in the repository by specifying "uses: ./.github/workflows/build_wheels.yml" +on: [workflow_dispatch, workflow_call] + +jobs: + get-version: + runs-on: ubuntu-latest + outputs: + release_version: ${{ steps.get_release_version.outputs.release_version }} + version_without_prefix: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + highest_semver_tag: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + steps: + - uses: actions/checkout@v2 + - name: Get release version + id: get_release_version + run: echo ::set-output name=release_version::${GITHUB_REF#refs/*/} + - name: Get release version without prefix + id: get_release_version_without_prefix + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + echo ::set-output name=version_without_prefix::${RELEASE_VERSION:1} + - name: Get highest semver + id: get_highest_semver + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + run: | + source infra/scripts/setup-common-functions.sh + SEMVER_REGEX='^v[0-9]+\.[0-9]+\.[0-9]+(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$' + if echo "${RELEASE_VERSION}" | grep -P "$SEMVER_REGEX" &>/dev/null ; then + echo ::set-output name=highest_semver_tag::$(get_tag_release -m) + fi + - name: Check output + env: + RELEASE_VERSION: ${{ steps.get_release_version.outputs.release_version }} + VERSION_WITHOUT_PREFIX: ${{ steps.get_release_version_without_prefix.outputs.version_without_prefix }} + HIGHEST_SEMVER_TAG: ${{ steps.get_highest_semver.outputs.highest_semver_tag }} + run: | + echo $RELEASE_VERSION + echo $VERSION_WITHOUT_PREFIX + echo $HIGHEST_SEMVER_TAG + + build-python-wheel: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ ubuntu-latest, macos-10.15 ] + steps: + - uses: actions/checkout@v2 + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' + - name: Build UI + run: make build-ui + - name: Build wheels + uses: pypa/cibuildwheel@v2.7.0 + env: + CIBW_BUILD: "cp3*_x86_64" + CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" + CIBW_ARCHS: "native" + CIBW_ENVIRONMENT: > + COMPILE_GO=True PATH=$PATH:/usr/local/go/bin + CIBW_BEFORE_ALL_LINUX: | + curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz + tar -C /usr/local -xzf go.tar.gz + go version + CIBW_BEFORE_ALL_MACOS: | + curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg + sudo installer -pkg python.pkg -target / + # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. + CIBW_BEFORE_BUILD: | + make install-protoc-dependencies + make install-go-proto-dependencies + make install-go-ci-dependencies + git status + git restore go.mod go.sum + CIBW_BEFORE_TEST: "cd {project} && git status" + # py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881. + - name: Build py310 specific wheels for macos + if: matrix.os == 'macos-10.15' + uses: pypa/cibuildwheel@v2.7.0 + env: + CIBW_BUILD: "cp310-macosx_x86_64" + CIBW_ARCHS: "native" + # Need this environment variable because of this issue: https://github.com/pypa/cibuildwheel/issues/952. + CIBW_ENVIRONMENT: > + _PYTHON_HOST_PLATFORM=macosx-10.15-x86_64 + # There's a `git restore` in here because remnant go.mod, go.sum changes from the build mess up the wheel naming. + CIBW_BEFORE_BUILD: | + git status + git restore go.mod go.sum + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/*.whl + + build-source-distribution: + name: Build source distribution + runs-on: macos-10.15 + steps: + - uses: actions/checkout@v2 + - name: Setup Python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + architecture: x64 + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: '17.x' + registry-url: 'https://registry.npmjs.org' + - name: Build and install dependencies + # There's a `git restore` in here because `make install-go-ci-dependencies` is actually messing up go.mod & go.sum. + run: | + pip install -U pip setuptools wheel twine + make install-protoc-dependencies + make install-go-proto-dependencies + make install-go-ci-dependencies + make build-ui + git status + git restore go.mod go.sum + - name: Build + run: | + python3 setup.py sdist + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: dist/* + + verify-python-wheels: + runs-on: ${{ matrix.os }} + needs: [build-python-wheel, build-source-distribution] + strategy: + matrix: + os: [ubuntu-latest, macos-10.15 ] + python-version: [ "3.7", "3.8", "3.9", "3.10"] + from-source: [ True, False ] + env: + # this script is for testing servers + # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself + TEST_SCRIPT: | + timeout 10s $@ & pid=$! + wait $pid + ret=$? + if [[ $ret -ne 124 ]] + then + exit $ret + else + echo "Succeeded!" + fi + steps: + - name: Setup Python + id: setup-python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + - uses: actions/setup-go@v3 + with: + go-version: '>=1.17.0' + - uses: actions/download-artifact@v2 + with: + name: wheels + path: dist + - name: Install wheel + if: ${{ !matrix.from-source }} + # try to install all wheels; only the current platform wheel should be actually installed + run: | + cd dist/ + pip install wheel + for f in *.whl; do pip install $f || true; done + - name: Install dist with go + if: ${{ matrix.from-source && (matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest')}} + env: + COMPILE_GO: "True" + run: | + pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 + pip install dist/*tar.gz + # py3.10 on MacOS does not work with Go so we have to install separately. Issue is tracked here: https://github.com/feast-dev/feast/issues/2881 + - name: Install dist w/o go + if: ${{ matrix.from-source && matrix.python-version == '3.10' && matrix.os == 'macos-10.15'}} + run: pip install dist/*tar.gz + - name: Install OS X dependencies + if: matrix.os == 'macos-10.15' + run: brew install coreutils + - name: Smoke test + run: | + feast init test_repo + cd test_repo/ + feast apply + echo "$TEST_SCRIPT" > run-and-wait.sh + bash run-and-wait.sh feast serve + bash run-and-wait.sh feast ui + # We disable this test for the Python 3.10 binary since it does not include Go. + - name: Smoke test with go + if: matrix.python-version != '3.10' || matrix.os == 'ubuntu-latest' + run: | + cd test_repo/ + feast apply + echo "$TEST_SCRIPT" > run-and-wait.sh + pip install cffi + printf "\ngo_feature_retrieval: True" >> feature_store.yaml + bash run-and-wait.sh feast serve \ No newline at end of file diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index a856fbe2cba..fc2bb523876 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -93,7 +93,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index a87542b9454..39593f02ce0 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -74,7 +74,7 @@ jobs: architecture: x64 - uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' architecture: 'x64' - uses: actions/cache@v2 with: diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml index b57126ca6c1..a0a6d7dd382 100644 --- a/.github/workflows/linter.yml +++ b/.github/workflows/linter.yml @@ -6,20 +6,20 @@ jobs: lint-python: runs-on: [ubuntu-latest] env: - PYTHON: 3.7 + PYTHON: 3.8 steps: - uses: actions/checkout@v2 - name: Setup Python id: setup-python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" architecture: x64 - name: Setup Go id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<22.1" @@ -54,12 +54,12 @@ jobs: id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 - name: Setup Python id: setup-python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Upgrade pip version run: | pip install --upgrade "pip>=21.3.1,<22.1" diff --git a/.github/workflows/master_only.yml b/.github/workflows/master_only.yml index 2042987617b..0cb49bb525c 100644 --- a/.github/workflows/master_only.yml +++ b/.github/workflows/master_only.yml @@ -13,7 +13,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Set up AWS SDK uses: aws-actions/configure-aws-credentials@v1 with: @@ -50,6 +52,7 @@ jobs: docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + --load \ . docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} outputs: @@ -60,7 +63,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" ] + python-version: [ "3.8", "3.9", "3.10" ] go-version: [ 1.17.0 ] os: [ ubuntu-latest ] env: @@ -140,7 +143,7 @@ jobs: SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: @@ -177,7 +180,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Login to DockerHub uses: docker/login-action@v1 with: diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 089d9f47336..e1c7ed2de2b 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -20,7 +20,7 @@ jobs: (github.event.action != 'labeled' && (contains(github.event.pull_request.labels.*.name, 'ok-to-test') || contains(github.event.pull_request.labels.*.name, 'approved') || contains(github.event.pull_request.labels.*.name, 'lgtm'))) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: # pull_request_target runs the workflow in the context of the base repo # as such actions/checkout needs to be explicit configured to retrieve @@ -30,7 +30,9 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 + with: + install: true - name: Set up AWS SDK uses: aws-actions/configure-aws-credentials@v1 with: @@ -67,6 +69,7 @@ jobs: docker build \ --file sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile \ --tag $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} \ + --load \ . docker push $ECR_REGISTRY/$ECR_REPOSITORY:${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }} outputs: @@ -81,7 +84,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.7" ] + python-version: [ "3.8" ] os: [ ubuntu-latest ] env: OS: ${{ matrix.os }} @@ -114,7 +117,7 @@ jobs: id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -167,7 +170,7 @@ jobs: SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index dad6eddc862..184fdb3cb6e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -125,9 +125,12 @@ jobs: - name: Publish Helm charts run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX + build_wheels: + uses: ./.github/workflows/build_wheels.yml + publish-python-sdk: runs-on: ubuntu-latest - needs: [verify-python-wheel] + needs: [build_wheels] steps: - uses: actions/download-artifact@v2 with: @@ -138,149 +141,6 @@ jobs: user: __token__ password: ${{ secrets.PYPI_PASSWORD }} - - verify-python-wheel: - runs-on: ${{ matrix.os }} - needs: [build-python-sdk, build-python-sdk-macos-py310] - strategy: - matrix: - os: [ ubuntu-latest, macos-10.15 ] - python-version: [ "3.7", "3.8", "3.9", "3.10"] - from-source: [ True, False ] - env: - # this script is for testing servers - # it starts server with timeout and checks whether process killed by timeout (started healthy) or died by itself - TEST_SCRIPT: | - timeout 10s $@ & pid=$! - wait $pid - ret=$? - if [[ $ret -ne 124 ]] - then - exit $ret - else - echo "Succeeded!" - fi - steps: - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - uses: actions/setup-go@v3 - with: - go-version: '>=1.17.0' - - uses: actions/download-artifact@v2 - with: - name: wheels - path: dist - - name: Install wheel - if: ${{ !matrix.from-source }} - # try to install all wheels; only the current platform wheel should be actually installed - run: | - cd dist/ - pip install wheel - for f in *.whl; do pip install $f || true; done - - name: Install sdist - if: ${{ matrix.from-source }} - env: - COMPILE_GO: "True" - run: | - pip install 'grpcio-tools==1.44.0' 'pybindgen==0.22.0' - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 - pip install dist/*tar.gz - - name: Install OS X dependencies - if: matrix.os == 'macos-10.15' - run: brew install coreutils - - name: Smoke test - run: | - feast init test_repo - cd test_repo/ - feast apply - echo "$TEST_SCRIPT" > run-and-wait.sh - bash run-and-wait.sh feast serve - bash run-and-wait.sh feast ui - - pip install cffi - printf "\ngo_feature_retrieval: True" >> feature_store.yaml - bash run-and-wait.sh feast serve - - build-python-sdk: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-latest, macos-10.15 ] - steps: - - uses: actions/checkout@v2 - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '17.x' - registry-url: 'https://registry.npmjs.org' - - name: Build UI - run: make build-ui - - name: Build wheels - uses: pypa/cibuildwheel@v2.4.0 - env: - CIBW_BUILD: "cp3*_x86_64" - CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" - CIBW_ARCHS: "native" - CIBW_ENVIRONMENT: > - COMPILE_GO=True PATH=$PATH:/usr/local/go/bin - CIBW_BEFORE_ALL_LINUX: | - curl -o go.tar.gz https://dl.google.com/go/go1.18.2.linux-amd64.tar.gz - tar -C /usr/local -xzf go.tar.gz - go version - CIBW_BEFORE_ALL_MACOS: | - curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg - sudo installer -pkg python.pkg -target / - CIBW_BEFORE_BUILD: | - make install-protoc-dependencies - make install-go-proto-dependencies - make install-go-ci-dependencies - - - uses: actions/upload-artifact@v2 - with: - name: wheels - path: ./wheelhouse/*.whl - - - build-python-sdk-macos-py310: - runs-on: macos-10.15 - env: - COMPILE_GO: True - steps: - - uses: actions/checkout@v2 - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - architecture: x64 - - name: Setup Node - uses: actions/setup-node@v2 - with: - node-version: '17.x' - registry-url: 'https://registry.npmjs.org' - - name: Build and install dependencies - run: | - pip install -U pip setuptools wheel twine - make install-protoc-dependencies - make install-go-proto-dependencies - make install-go-ci-dependencies - make build-ui - - name: Build - run: | - python3 setup.py sdist bdist_wheel - - - uses: actions/upload-artifact@v2 - with: - name: wheels - path: dist/* - - publish-java-sdk: container: maven:3.6-jdk-11 runs-on: ubuntu-latest diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3573fe26568..a9bf3deba32 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -7,11 +7,9 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest] + python-version: [ "3.8", "3.9", "3.10" ] + os: [ ubuntu-latest, macOS-latest ] exclude: - - os: macOS-latest - python-version: "3.8" - os: macOS-latest python-version: "3.9" - os: macOS-latest @@ -31,10 +29,15 @@ jobs: id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 + - name: Install mysql on macOS + if: startsWith(matrix.os, 'macOS') + run: | + brew install mysql + PATH=$PATH:/usr/local/mysql/bin - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=22.1,<23" - name: Get pip cache dir id: pip-cache run: | @@ -55,12 +58,13 @@ jobs: run: make install-python-ci-dependencies - name: Test Python env: + IS_TEST: "True" SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests + run: FEAST_USAGE=False pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: @@ -79,15 +83,15 @@ jobs: id: setup-python uses: actions/setup-python@v2 with: - python-version: "3.7" + python-version: "3.8" - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1,<22.1" + pip install --upgrade "pip>=22.1,<23" - name: Setup Go id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 - name: Install dependencies run: make install-go-proto-dependencies - name: Compile protos diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 00000000000..b28dfbe49f5 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,43 @@ +# https://www.gitpod.io/docs/config-gitpod-file +tasks: + - init: | + python -m venv venv + source venv/bin/activate + + pip install pre-commit + pre-commit install --hook-type pre-commit --hook-type pre-push + pip install '.[dev]' + make compile-protos-python + make compile-protos-go + make compile-go-lib + env: + PYTHONUSERBASE: "/workspace/.pip-modules" + command: | + source venv/bin/activate + + git config --global alias.ci 'commit -s' + git config --global alias.sw switch + git config --global alias.st status + git config --global alias.co checkout + git config --global alias.br branch + git config --global alias.df diff +github: + prebuilds: + # enable for the default branch (defaults to true) + master: true + # enable for all branches in this repo (defaults to false) + branches: false + # enable for pull requests coming from this repo (defaults to true) + pullRequests: true + # enable for pull requests coming from forks (defaults to false) + pullRequestsFromForks: false + # add a check to pull requests (defaults to true) + addCheck: true + # add a "Review in Gitpod" button as a comment to pull requests (defaults to false) + addComment: false + # add a "Review in Gitpod" button to the pull request's description (defaults to false) + addBadge: false + +vscode: + extensions: + - ms-python.python diff --git a/.readthedocs.yml b/.readthedocs.yml index dea27e20b3a..75499aa5ddb 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -7,6 +7,6 @@ formats: - pdf python: - version: 3.7 + version: "3.8" install: - requirements: sdk/python/docs/requirements.txt \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ef9d0ec369..bd7e8098f36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,73 @@ # Changelog +# [0.22.0](https://github.com/feast-dev/feast/compare/v0.21.0...v0.22.0) (2022-06-29) + + +### Bug Fixes + +* Add columns for user metadata in the tables ([#2760](https://github.com/feast-dev/feast/issues/2760)) ([269055e](https://github.com/feast-dev/feast/commit/269055e25178956715c163547c9f0a33a5892a75)) +* Add project columns in the SQL Registry ([#2784](https://github.com/feast-dev/feast/issues/2784)) ([336fdd1](https://github.com/feast-dev/feast/commit/336fdd104d2669d19ca56b0d17eadb78fc95a3cd)) +* Add S3FS dependency (which Dask depends on for S3 files) ([#2701](https://github.com/feast-dev/feast/issues/2701)) ([5d6fa94](https://github.com/feast-dev/feast/commit/5d6fa9408052695dfce179ea619d8362898d6329)) +* Bugfixes for how registry is loaded ([#2768](https://github.com/feast-dev/feast/issues/2768)) ([ecb8b2a](https://github.com/feast-dev/feast/commit/ecb8b2af4ba0f9d52be8ac560ac2a9da0f46f38e)) +* Conversion of null timestamp from proto to python ([#2814](https://github.com/feast-dev/feast/issues/2814)) ([cb23648](https://github.com/feast-dev/feast/commit/cb23648da71cbf63e382ec75a8afe7228ff8cbc2)) +* Correct feature statuses during feature logging test ([#2709](https://github.com/feast-dev/feast/issues/2709)) ([cebf609](https://github.com/feast-dev/feast/commit/cebf609309f81a0f4ccded3448cefac5923be525)) +* Correctly generate projects-list.json when calling feast ui and using postgres as a source ([#2845](https://github.com/feast-dev/feast/issues/2845)) ([bee8076](https://github.com/feast-dev/feast/commit/bee8076085e9f42e783fb9ce5ad07b618d913f0d)) +* Dynamodb drops missing entities when batching ([#2802](https://github.com/feast-dev/feast/issues/2802)) ([a2e9209](https://github.com/feast-dev/feast/commit/a2e9209f7a7283925b56b5877e8fdc2e2d863798)) +* Enable faulthandler and disable flaky tests ([#2815](https://github.com/feast-dev/feast/issues/2815)) ([4934d84](https://github.com/feast-dev/feast/commit/4934d843fc65dd62ba1c1302243c1b2c151c78d2)) +* Explicitly translate errors when instantiating the go fs ([#2842](https://github.com/feast-dev/feast/issues/2842)) ([7a2c4cd](https://github.com/feast-dev/feast/commit/7a2c4cd8bf9b16331fad1d2e7d0ea3f85589f96b)) +* Fix broken roadmap links ([#2690](https://github.com/feast-dev/feast/issues/2690)) ([b3ba8aa](https://github.com/feast-dev/feast/commit/b3ba8aaf3a87343d756a2996376865096d543515)) +* Fix bugs in applying stream feature view and retrieving online features ([#2754](https://github.com/feast-dev/feast/issues/2754)) ([d024e5e](https://github.com/feast-dev/feast/commit/d024e5efac085ec12a17005389229bfd93cf466e)) +* Fix Feast UI failure with new way of specifying entities ([#2773](https://github.com/feast-dev/feast/issues/2773)) ([0d1ac01](https://github.com/feast-dev/feast/commit/0d1ac01ef45a1ee78799c7c5ebf30b2476bfc30e)) +* Fix feature view __getitem__ for feature services ([#2769](https://github.com/feast-dev/feast/issues/2769)) ([88cc47d](https://github.com/feast-dev/feast/commit/88cc47dea936f34c0798b6c6c31fda0d1b3ecbd7)) +* Fix issue when user specifies a port for feast ui ([#2692](https://github.com/feast-dev/feast/issues/2692)) ([1c621fe](https://github.com/feast-dev/feast/commit/1c621fe3649900a59e85fe9c4f3840dd09bc88d0)) +* Fix macos wheel version for 310 and also checkout edited go files ([#2890](https://github.com/feast-dev/feast/issues/2890)) ([bdf170f](https://github.com/feast-dev/feast/commit/bdf170f37971abc34930d62559374882139d69b6)) +* Fix on demand feature view crash from inference when it uses df.apply ([#2713](https://github.com/feast-dev/feast/issues/2713)) ([c5539fd](https://github.com/feast-dev/feast/commit/c5539fd9874fed3f69b0aaebc4d1d32e624bd041)) +* Fix SparkKafkaProcessor `query_timeout` parameter ([#2789](https://github.com/feast-dev/feast/issues/2789)) ([a8d282d](https://github.com/feast-dev/feast/commit/a8d282d3e4f041824ef7479f22c306dbfb8ad569)) +* Fix workflow syntax error ([#2869](https://github.com/feast-dev/feast/issues/2869)) ([fae45a1](https://github.com/feast-dev/feast/commit/fae45a11e78c23a58012d1d3cd2b9eb20a267794)) +* Fixed custom S3 endpoint read fail ([#2786](https://github.com/feast-dev/feast/issues/2786)) ([6fec431](https://github.com/feast-dev/feast/commit/6fec431dd5c9d53a678f58c9b87e2b1cdb02b238)) +* Go install gopy instead using go mod tidy ([#2863](https://github.com/feast-dev/feast/issues/2863)) ([2f2b519](https://github.com/feast-dev/feast/commit/2f2b5192f5f9c3e38183a2912f3bc7e754c9db8a)) +* Hydrate infra object in the sql registry proto() method ([#2782](https://github.com/feast-dev/feast/issues/2782)) ([452dcd3](https://github.com/feast-dev/feast/commit/452dcd31da195273ef80ee39db5a7893b7d48cc2)) +* Implement apply_materialization and infra methods in sql registry ([#2775](https://github.com/feast-dev/feast/issues/2775)) ([4ed107c](https://github.com/feast-dev/feast/commit/4ed107cdf6476faf20a4e09716ade87cb99f1d14)) +* Minor refactor to format exception message ([#2764](https://github.com/feast-dev/feast/issues/2764)) ([da763c6](https://github.com/feast-dev/feast/commit/da763c6766cd7bec801312849e884b9dc8f8fb51)) +* Prefer installing gopy from feast's fork as opposed to upstream ([#2839](https://github.com/feast-dev/feast/issues/2839)) ([34c997d](https://github.com/feast-dev/feast/commit/34c997d81b0084d81fb6fb21d5d4374fc7760695)) +* Python server is not correctly starting in integration tests ([#2706](https://github.com/feast-dev/feast/issues/2706)) ([7583a0b](https://github.com/feast-dev/feast/commit/7583a0b1840c663af50bf4382c3ff2368593bb9c)) +* Random port allocation for python server in tests ([#2710](https://github.com/feast-dev/feast/issues/2710)) ([dee8090](https://github.com/feast-dev/feast/commit/dee80908ffb247367526561de3ad4841516a3196)) +* Refactor test to reuse LocalRegistryFile ([#2763](https://github.com/feast-dev/feast/issues/2763)) ([4339c0a](https://github.com/feast-dev/feast/commit/4339c0acc990de2b00db90221f41ac0d33e68544)) +* Revert "chore(release): release 0.22.0" ([#2852](https://github.com/feast-dev/feast/issues/2852)) ([e6a4636](https://github.com/feast-dev/feast/commit/e6a463666e53b87c9d41334f6151df5e2c09c805)) +* Stop running go mod tidy in setup.py ([#2877](https://github.com/feast-dev/feast/issues/2877)) ([676ecbb](https://github.com/feast-dev/feast/commit/676ecbb31550275017d83a6bc8dbf1e03f0d24fa)), closes [/github.com/pypa/cibuildwheel/issues/189#issuecomment-549933912](https://github.com//github.com/pypa/cibuildwheel/issues/189/issues/issuecomment-549933912) +* Support push sources in stream feature views ([#2704](https://github.com/feast-dev/feast/issues/2704)) ([0d60eaa](https://github.com/feast-dev/feast/commit/0d60eaa0b7f32c84eb015c814a3c879e7d4f50fd)) +* Sync publish and build_wheels workflow to fix verify wheel error. ([#2871](https://github.com/feast-dev/feast/issues/2871)) ([b0f050a](https://github.com/feast-dev/feast/commit/b0f050a31946f9ff43ac3a3523d4dbd2a9802cc3)) +* Update roadmap with stream feature view rfc ([#2824](https://github.com/feast-dev/feast/issues/2824)) ([fc8f890](https://github.com/feast-dev/feast/commit/fc8f89059d1095e81e265f342ccaa26ff7f444f9)) +* Update udf tests and add base functions to streaming fcos and fix some nonetype errors ([#2776](https://github.com/feast-dev/feast/issues/2776)) ([331a214](https://github.com/feast-dev/feast/commit/331a214b04dc8b3f9347e79819130fa7bdf9e4c3)) + + +### Features + +* Add feast repo-upgrade for automated repo upgrades ([#2733](https://github.com/feast-dev/feast/issues/2733)) ([a3304d4](https://github.com/feast-dev/feast/commit/a3304d4e2d6d803f2a0fe35ef74204bd5cef7517)) +* Add file write_to_offline_store functionality ([#2808](https://github.com/feast-dev/feast/issues/2808)) ([c0e2ad7](https://github.com/feast-dev/feast/commit/c0e2ad7bf7310289ec6a7a6bd8cd2f766786b0cd)) +* Add http endpoint to the Go feature server ([#2658](https://github.com/feast-dev/feast/issues/2658)) ([3347a57](https://github.com/feast-dev/feast/commit/3347a57240cd485b7572777d7b977869140ccb20)) +* Add simple TLS support in Go RedisOnlineStore ([#2860](https://github.com/feast-dev/feast/issues/2860)) ([521488d](https://github.com/feast-dev/feast/commit/521488d71fa5050f64aa04a8ba7ef9891a57ca94)) +* Add StreamProcessor and SparkKafkaProcessor as contrib ([#2777](https://github.com/feast-dev/feast/issues/2777)) ([83ab682](https://github.com/feast-dev/feast/commit/83ab682c14a11a92121866409bddc787021d52e5)) +* Added Spark support for Delta and Avro ([#2757](https://github.com/feast-dev/feast/issues/2757)) ([7d16516](https://github.com/feast-dev/feast/commit/7d1651687a474850ebb16d4c9c0ff4a3daa6d486)) +* CLI interface for validation of logged features ([#2718](https://github.com/feast-dev/feast/issues/2718)) ([c8b11b3](https://github.com/feast-dev/feast/commit/c8b11b3b790b60e916d3257a036b9cb6430f4685)) +* Enable stream feature view materialization ([#2798](https://github.com/feast-dev/feast/issues/2798)) ([a06700d](https://github.com/feast-dev/feast/commit/a06700dd81c9893e98f6709b82a6faa32be49915)) +* Enable stream feature view materialization ([#2807](https://github.com/feast-dev/feast/issues/2807)) ([7d57724](https://github.com/feast-dev/feast/commit/7d57724dd4ff7d5ca4549bac8c72dbd71c57fce7)) +* Implement `offline_write_batch` for BigQuery and Snowflake ([#2840](https://github.com/feast-dev/feast/issues/2840)) ([97444e4](https://github.com/feast-dev/feast/commit/97444e439d0dc7a66b7121161c6f6560ae53d307)) +* Offline push endpoint for pushing to offline stores ([#2837](https://github.com/feast-dev/feast/issues/2837)) ([a88cd30](https://github.com/feast-dev/feast/commit/a88cd30f7925005db6f7c400b391d5e73d1b00f6)) +* Push to Redshift batch source offline store directly ([#2819](https://github.com/feast-dev/feast/issues/2819)) ([5748a8b](https://github.com/feast-dev/feast/commit/5748a8bbe338dfcb3fbbdc59fb1f57a99e1ea5eb)) +* Scaffold for unified push api ([#2796](https://github.com/feast-dev/feast/issues/2796)) ([1bd0930](https://github.com/feast-dev/feast/commit/1bd093028785ac9349be56c9ea98a3bd94c47fbe)) +* SQLAlchemy Registry Support ([#2734](https://github.com/feast-dev/feast/issues/2734)) ([b3fe39c](https://github.com/feast-dev/feast/commit/b3fe39c1600fa370f28c7b01e2b3f7da716449c1)) +* Stream Feature View FCOS ([#2750](https://github.com/feast-dev/feast/issues/2750)) ([0cf3c92](https://github.com/feast-dev/feast/commit/0cf3c923717f561d5656c57eb0b61fcd569917bd)) +* Update stream fcos to have watermark and sliding interval ([#2765](https://github.com/feast-dev/feast/issues/2765)) ([3256952](https://github.com/feast-dev/feast/commit/325695275da610cecf2b9e820fd71f7f04179ccf)) +* Validating logged features via Python SDK ([#2640](https://github.com/feast-dev/feast/issues/2640)) ([2874fc5](https://github.com/feast-dev/feast/commit/2874fc5c85810a65f750377d34418c71e747110e)) + + +### Reverts + +* Revert "chore(release): release 0.22.0" (#2891) ([e5abf58](https://github.com/feast-dev/feast/commit/e5abf589020b3c261ac0ce38d295ba96daf317c2)), closes [#2891](https://github.com/feast-dev/feast/issues/2891) +* Revert "chore(release): release 0.22.0" (#2870) ([ffb0892](https://github.com/feast-dev/feast/commit/ffb089241d6521caa3be4034e6ae44af7dc4f8af)), closes [#2870](https://github.com/feast-dev/feast/issues/2870) +* Revert "Create main.yml" (#2867) ([47922a4](https://github.com/feast-dev/feast/commit/47922a4cda532871eecd5e17edef6d08a4a50110)), closes [#2867](https://github.com/feast-dev/feast/issues/2867) + # [0.21.0](https://github.com/feast-dev/feast/compare/v0.20.0...v0.21.0) (2022-05-13) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cce520d346..4bd14d762a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -75,7 +75,8 @@ Note that this means if you are midway through working through a PR and rebase, ## Feast Python SDK / CLI ### Environment Setup Setting up your development environment for Feast Python SDK / CLI: -1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing. +1. Ensure that you have Docker installed in your environment. Docker is used to provision service dependencies during testing, and build images for feature servers and other components. + 1. Please note that we use [Docker with BuiltKit](https://docs.docker.com/develop/develop-images/build_enhancements/). 2. Ensure that you have `make`, Python (3.7 and above) with `pip`, installed. 3. _Recommended:_ Create a virtual environment to isolate development dependencies to be installed ```sh diff --git a/Makefile b/Makefile index e208ab3c451..88f04aa95d8 100644 --- a/Makefile +++ b/Makefile @@ -47,7 +47,7 @@ package-protos: cp -r ${ROOT_DIR}/protos ${ROOT_DIR}/sdk/python/feast/protos compile-protos-python: - python setup.py build_python_protos + python setup.py build_python_protos --inplace install-python: python -m piptools sync sdk/python/requirements/py$(PYTHON)-requirements.txt @@ -170,10 +170,12 @@ install-go-proto-dependencies: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1.0 install-go-ci-dependencies: - # ToDo: currently gopy installation doesn't work w/o explicit go get in the next line - # ToDo: there should be a better way to install gopy - go get github.com/go-python/gopy + # TODO: currently gopy installation doesn't work w/o explicit go get in the next line + # TODO: there should be a better way to install gopy + go get github.com/go-python/gopy@v0.4.0 go install golang.org/x/tools/cmd/goimports + # The `go get` command on the previous lines download the lib along with replacing the dep to `feast-dev/gopy` + # but the following command is needed to install it for some reason. go install github.com/go-python/gopy python -m pip install pybindgen==0.22.0 @@ -206,31 +208,31 @@ push-ci-docker: # TODO(adchia): consider removing. This doesn't run successfully right now build-ci-docker: - docker build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile . + docker buildx build -t $(REGISTRY)/feast-ci:$(VERSION) -f infra/docker/ci/Dockerfile --load . push-feature-server-python-aws-docker: docker push $(REGISTRY)/feature-server-python-aws:$$VERSION build-feature-server-python-aws-docker: - docker build --build-arg VERSION=$$VERSION \ + docker buildx build --build-arg VERSION=$$VERSION \ -t $(REGISTRY)/feature-server-python-aws:$$VERSION \ - -f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile . + -f sdk/python/feast/infra/feature_servers/aws_lambda/Dockerfile --load . push-feature-transformation-server-docker: docker push $(REGISTRY)/feature-transformation-server:$(VERSION) build-feature-transformation-server-docker: - docker build --build-arg VERSION=$(VERSION) \ + docker buildx build --build-arg VERSION=$(VERSION) \ -t $(REGISTRY)/feature-transformation-server:$(VERSION) \ - -f sdk/python/feast/infra/transformation_servers/Dockerfile . + -f sdk/python/feast/infra/transformation_servers/Dockerfile --load . push-feature-server-java-docker: docker push $(REGISTRY)/feature-server-java:$(VERSION) build-feature-server-java-docker: - docker build --build-arg VERSION=$(VERSION) \ + docker buildx build --build-arg VERSION=$(VERSION) \ -t $(REGISTRY)/feature-server-java:$(VERSION) \ - -f java/infra/docker/feature-server/Dockerfile . + -f java/infra/docker/feature-server/Dockerfile --load . # Documentation diff --git a/README.md b/README.md index 5905bbd647f..0f77fbd42ca 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,6 @@ The list below contains the functionality that contributors are planning to deve * Items below that are in development (or planned for development) will be indicated in parentheses. * We welcome contribution to all items in the roadmap! -* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag\_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). * **Data Sources** @@ -172,17 +171,16 @@ The list below contains the functionality that contributors are planning to deve * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) + * [x] [Cassandra / AstraDB](https://github.com/datastaxdevs/feast-cassandra-online-store) * [ ] Bigtable (in progress) - * [ ] Cassandra -* **Streaming** - * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion](https://docs.feast.dev/reference/data-sources/push.md) - * [ ] Streaming ingestion on AWS - * [ ] Streaming ingestion on GCP * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) - * [ ] Streaming transformation +* **Streaming** + * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) + * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) @@ -196,7 +194,7 @@ The list below contains the functionality that contributors are planning to deve * [ ] Java Client * [ ] Go Client * [ ] Delete API - * [ ] Feature Logging (for training) + * [] Feature Logging (for training) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * [ ] Training-serving skew detection (in progress) @@ -207,9 +205,8 @@ The list below contains the functionality that contributors are planning to deve * [x] CLI for browsing feature registry * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) - * [x] Feast Web UI (Alpha release. See [documentation](https://docs.feast.dev/reference/alpha-web-ui.md)) + * [x] Feast Web UI (Alpha release. See [documentation](https://docs.feast.dev/reference/alpha-web-ui)) * [ ] REST API for browsing feature registry - * [ ] Feature versioning ## 🎓 Important Resources diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index f6f12e04d04..23049455e33 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -11,11 +11,13 @@ * [Concepts](getting-started/concepts/README.md) * [Overview](getting-started/concepts/overview.md) * [Data source](getting-started/concepts/data-source.md) + * [Dataset](getting-started/concepts/dataset.md) * [Entity](getting-started/concepts/entity.md) * [Feature view](getting-started/concepts/feature-view.md) + * [Stream feature view](getting-started/concepts/stream-feature-view.md) * [Feature retrieval](getting-started/concepts/feature-retrieval.md) * [Point-in-time joins](getting-started/concepts/point-in-time-joins.md) - * [Dataset](getting-started/concepts/dataset.md) + * [Registry](getting-started/concepts/registry.md) * [Architecture](getting-started/architecture-and-components/README.md) * [Overview](getting-started/architecture-and-components/overview.md) * [Feature repository](getting-started/architecture-and-components/feature-repository.md) @@ -23,6 +25,7 @@ * [Offline store](getting-started/architecture-and-components/offline-store.md) * [Online store](getting-started/architecture-and-components/online-store.md) * [Provider](getting-started/architecture-and-components/provider.md) +* [Learning by example](getting-started/feast-workshop.md) * [Third party integrations](getting-started/third-party-integrations.md) * [FAQ](getting-started/faq.md) @@ -34,6 +37,8 @@ * [Real-time credit scoring on AWS](tutorials/real-time-credit-scoring-on-aws.md) * [Driver stats on Snowflake](tutorials/driver-stats-on-snowflake.md) * [Validating historical features with Great Expectations](tutorials/validating-historical-features.md) +* [Using Scalable Registry](tutorials/using-scalable-registry.md) +* [Building streaming features](tutorials/building-streaming-features.md) ## How-to Guides @@ -60,6 +65,8 @@ * [BigQuery](reference/data-sources/bigquery.md) * [Redshift](reference/data-sources/redshift.md) * [Push](reference/data-sources/push.md) + * [Kafka](reference/data-sources/kafka.md) + * [Kinesis](reference/data-sources/kinesis.md) * [Spark (contrib)](reference/data-sources/spark.md) * [PostgreSQL (contrib)](reference/data-sources/postgres.md) * [Offline stores](reference/offline-stores/README.md) diff --git a/docs/getting-started/architecture-and-components/offline-store.md b/docs/getting-started/architecture-and-components/offline-store.md index 96914db9d07..29a72bd5f0f 100644 --- a/docs/getting-started/architecture-and-components/offline-store.md +++ b/docs/getting-started/architecture-and-components/offline-store.md @@ -13,3 +13,5 @@ It is not possible to query all data sources from all offline stores, and only a Please see the [Offline Stores](../../reference/offline-stores/) reference for more details on configuring offline stores. +Please see the [Push Source](reference/data-sources/push.md) for reference on how to push features directly to the offline store in your feature store. + diff --git a/docs/getting-started/architecture-and-components/online-store.md b/docs/getting-started/architecture-and-components/online-store.md index 4f2e44c92ce..21b4dbcb9c7 100644 --- a/docs/getting-started/architecture-and-components/online-store.md +++ b/docs/getting-started/architecture-and-components/online-store.md @@ -12,4 +12,4 @@ Once the above data source is materialized into Feast \(using `feast materialize ![](../../.gitbook/assets/image%20%285%29.png) -Features can also be written to the online store via [push sources](https://docs.feast.dev/reference/data-sources/push) \ No newline at end of file +Features can also be written to the online store via [push sources](../../reference/data-sources/push.md) \ No newline at end of file diff --git a/docs/getting-started/architecture-and-components/overview.md b/docs/getting-started/architecture-and-components/overview.md index bf5c12dcc20..0c47fb2753d 100644 --- a/docs/getting-started/architecture-and-components/overview.md +++ b/docs/getting-started/architecture-and-components/overview.md @@ -23,7 +23,7 @@ A complete Feast deployment contains the following components: * Materialize (load) feature values into the online store. * Build and retrieve training datasets from the offline store. * Retrieve online features. -* **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is populated by materialization jobs and from [stream ingestion](../../reference/alpha-stream-ingestion.md). +* **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is populated by materialization jobs and from [stream ingestion](../../reference/data-sources/push.md). * **Offline Store:** The offline store persists batch data that has been ingested into Feast. This data is used for producing training datasets. Feast does not manage the offline store directly, but runs queries against it. {% hint style="info" %} diff --git a/docs/getting-started/concepts/README.md b/docs/getting-started/concepts/README.md index e7b29eb0047..6f2f64955dc 100644 --- a/docs/getting-started/concepts/README.md +++ b/docs/getting-started/concepts/README.md @@ -4,12 +4,18 @@ {% page-ref page="data-source.md" %} +{% page-ref page="dataset.md" %} + {% page-ref page="entity.md" %} {% page-ref page="feature-view.md" %} +{% page-ref page="feature-view.md" %} + +{% page-ref page="stream-feature-view.md" %} + {% page-ref page="feature-retrieval.md" %} {% page-ref page="point-in-time-joins.md" %} -{% page-ref page="dataset.md" %} +{% page-ref page="registry.md" %} \ No newline at end of file diff --git a/docs/getting-started/concepts/registry.md b/docs/getting-started/concepts/registry.md new file mode 100644 index 00000000000..2236f319312 --- /dev/null +++ b/docs/getting-started/concepts/registry.md @@ -0,0 +1,9 @@ +# Registry + +The Feast registry is where all applied Feast objects (e.g. Feature views, entities, etc) are stored. The registry exposes methods to apply, list, retrieve and delete these objects. The registry is abstraction, with multiple possible implementations. + +By default, the registry Feast uses a file-based registry implementation, which stores the protobuf representation of the registry as a serialized file. This registry file can be stored in a local file system, or in cloud storage (in, say, S3 or GCS). + +However, there's inherent limitations with a file-based registry, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +Alternatively, a [SQL Registry](../../tutorials/using-scalable-registry.md) can be used for a more scalable registry. \ No newline at end of file diff --git a/docs/getting-started/concepts/stream-feature-view.md b/docs/getting-started/concepts/stream-feature-view.md new file mode 100644 index 00000000000..2ce39936145 --- /dev/null +++ b/docs/getting-started/concepts/stream-feature-view.md @@ -0,0 +1,56 @@ +# Stream feature view + +## Stream feature views + +A stream feature view is an extension of a normal feature view. The primary difference is that stream feature views have both stream and batch data sources, whereas a normal feature view only has a batch data source. + +Stream feature views should be used instead of normal feature views when there are stream data sources (e.g. Kafka and Kinesis) available to provide fresh features in an online setting. Here is an example definition of a stream feature view with an attached transformation: + +```python +from datetime import timedelta + +from feast import Field, FileSource, KafkaSource, stream_feature_view +from feast.data_format import JsonFormat +from feast.types import Float32 + +driver_stats_batch_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", +) + +driver_stats_stream_source = KafkaSource( + name="driver_stats_stream", + kafka_bootstrap_servers="localhost:9092", + topic="drivers", + timestamp_field="event_timestamp", + batch_source=driver_stats_batch_source, + message_format=JsonFormat( + schema_json="driver_id integer, event_timestamp timestamp, conv_rate double, acc_rate double, created timestamp" + ), + watermark_delay_threshold=timedelta(minutes=5), +) + +@stream_feature_view( + entities=[driver], + ttl=timedelta(seconds=8640000000), + mode="spark", + schema=[ + Field(name="conv_percentage", dtype=Float32), + Field(name="acc_percentage", dtype=Float32), + ], + timestamp_field="event_timestamp", + online=True, + source=driver_stats_stream_source, +) +def driver_hourly_stats_stream(df: DataFrame): + from pyspark.sql.functions import col + + return ( + df.withColumn("conv_percentage", col("conv_rate") * 100.0) + .withColumn("acc_percentage", col("acc_rate") * 100.0) + .drop("conv_rate", "acc_rate") + ) +``` + +See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to use stream feature views. diff --git a/docs/getting-started/feast-workshop.md b/docs/getting-started/feast-workshop.md new file mode 100644 index 00000000000..c883625dac9 --- /dev/null +++ b/docs/getting-started/feast-workshop.md @@ -0,0 +1,42 @@ +# Learning by example + +This workshop aims to teach users about Feast. + +We explain concepts & best practices by example, and also showcase how to address common use cases. + +### Pre-requisites + +This workshop assumes you have the following installed: + +* A local development environment that supports running Jupyter notebooks (e.g. VSCode with Jupyter plugin) +* Python 3.7+ +* Java 11 (for Spark, e.g. `brew install java11`) +* pip +* Docker & Docker Compose (e.g. `brew install docker docker-compose`) +* Terraform ([docs](https://learn.hashicorp.com/tutorials/terraform/install-cli#install-terraform)) +* AWS CLI +* An AWS account setup with credentials via `aws configure` (e.g see [AWS credentials quickstart](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html#cli-configure-quickstart-creds)) + +Since we'll be learning how to leverage Feast in CI/CD, you'll also need to fork this workshop repository. + +#### **Caveats** + +* M1 Macbook development is untested with this flow. See also [How to run / develop for Feast on M1 Macs](https://github.com/feast-dev/feast/issues/2105). +* Windows development has only been tested with WSL. You will need to follow this [guide](https://docs.docker.com/desktop/windows/wsl/) to have Docker play nicely. + +### Modules + +_See also:_ [_Feast quickstart_](https://docs.feast.dev/getting-started/quickstart)_,_ [_Feast x Great Expectations tutorial_](https://docs.feast.dev/tutorials/validating-historical-features) + +These are meant mostly to be done in order, with examples building on previous concepts. + +| Time (min) | Description | Module | +| :--------: | ----------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| 30-45 | Setting up Feast projects & CI/CD + powering batch predictions | [Module 0](https://github.com/feast-dev/feast-workshop/tree/main/module\_0) | +| 15-20 | Streaming ingestion & online feature retrieval with Kafka, Spark, Redis | [Module 1](https://github.com/feast-dev/feast-workshop/tree/main/module\_1) | +| 10-15 | Real-time feature engineering with on demand transformations | [Module 2](https://github.com/feast-dev/feast-workshop/tree/main/module\_2) | +| TBD | Feature server deployment (embed, as a service, AWS Lambda) | TBD | +| TBD | Versioning features / models in Feast | TBD | +| TBD | Data quality monitoring in Feast | TBD | +| TBD | Batch transformations | TBD | +| TBD | Stream transformations | TBD | diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 41449b77e39..b5fe7bad4b9 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -125,8 +125,6 @@ driver_stats_fs = FeatureService( {% endtab %} {% endtabs %} -![Demo parquet data: data/driver\_stats.parquet](../.gitbook/assets/screen-shot-2021-08-23-at-2.35.18-pm.png) - The key line defining the overall architecture of the feature store is the **provider**. This defines where the raw data exists (for generating training data & feature values for serving), and where to materialize feature values to in the online store (for serving). Valid values for `provider` in `feature_store.yaml` are: @@ -139,6 +137,16 @@ Note that there are many other sources Feast works with, including Azure, Hive, A custom setup can also be made by following [adding a custom provider](../how-to-guides/creating-a-custom-provider.md). +### Inspecting the raw data + +The raw feature data we have in this demo is stored in a local parquet file. The dataset captures hourly stats of a driver in a ride-sharing app. + +```python +import pandas as pd +pd.read_parquet("data/driver_stats.parquet") +``` + +![Demo parquet data: data/driver\_stats.parquet](../.gitbook/assets/screen-shot-2021-08-23-at-2.35.18-pm.png) ## Step 3: Register feature definitions and deploy your feature store @@ -367,7 +375,15 @@ pprint(feature_vector) ## Step 7: Using a feature service to fetch online features instead. -You can also use feature services to manage multiple features, and decouple feature view definitions and the features needed by end applications. The feature store can also be used to fetch either online or historical features using the same api below. More information can be found [here](https://docs.feast.dev/getting-started/concepts/feature-service). +You can also use feature services to manage multiple features, and decouple feature view definitions and the features needed by end applications. The feature store can also be used to fetch either online or historical features using the same api below. More information can be found [here](https://docs.feast.dev/getting-started/concepts/feature-retrieval). + +The `driver_activity` feature service pulls all features from the `driver_hourly_stats` feature view: + +```python +driver_stats_fs = FeatureService( + name="driver_activity", features=[driver_hourly_stats_view] +) +``` {% tabs %} {% tab title="Python" %} @@ -376,7 +392,7 @@ from feast import FeatureStore feature_store = FeatureStore('.') # Initialize the feature store feature_service = feature_store.get_feature_service("driver_activity") -features = feature_store.get_online_features( +feature_vector = feature_store.get_online_features( features=feature_service, entity_rows=[ # {join_key: entity_value} @@ -384,6 +400,7 @@ features = feature_store.get_online_features( {"driver_id": 1005}, ], ).to_dict() +pprint(feature_vector) ``` {% tabs %} diff --git a/docs/getting-started/third-party-integrations.md b/docs/getting-started/third-party-integrations.md index ab926682662..8a862891f8d 100644 --- a/docs/getting-started/third-party-integrations.md +++ b/docs/getting-started/third-party-integrations.md @@ -46,8 +46,8 @@ Don't see your offline store or online store of choice here? Check out our guide * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) +* [x] [Cassandra / AstraDB](https://github.com/datastaxdevs/feast-cassandra-online-store) * [ ] Bigtable (in progress) -* [ ] Cassandra ### **Deployments** diff --git a/docs/how-to-guides/running-feast-in-production.md b/docs/how-to-guides/running-feast-in-production.md index 53808326091..6023c5ac66e 100644 --- a/docs/how-to-guides/running-feast-in-production.md +++ b/docs/how-to-guides/running-feast-in-production.md @@ -3,15 +3,15 @@ ## Overview After learning about Feast concepts and playing with Feast locally, you're now ready to use Feast in production. -This guide aims to help with the transition from a sandbox project to production-grade deployment in the cloud or on-premise. +This guide aims to help with the transition from a sandbox project to production-grade deployment in the cloud or on-premise. Overview of typical production configuration is given below: ![Overview](production-simple.png) {% hint style="success" %} -**Important note:** We're trying to keep Feast modular. With the exception of the core, most of the Feast blocks are loosely connected and can be used independently. Hence, you are free to build your own production configuration. -For example, you might not have a stream source and, thus, no need to write features in real-time to an online store. +**Important note:** We're trying to keep Feast modular. With the exception of the core, most of the Feast blocks are loosely connected and can be used independently. Hence, you are free to build your own production configuration. +For example, you might not have a stream source and, thus, no need to write features in real-time to an online store. Or you might not need to retrieve online features. Furthermore, there's no single "true" approach. As you will see in this guide, Feast usually provides several options for each problem. @@ -95,7 +95,7 @@ In summary, once you have set up a Git based repository with CI that runs `feast To keep your online store up to date, you need to run a job that loads feature data from your feature view sources into your online store. In Feast, this loading operation is called materialization. -### 2.1. Manual materializations +### 2.1. Manual materializations The simplest way to schedule materialization is to run an **incremental** materialization using the Feast CLI: ```text @@ -116,7 +116,7 @@ In the above example we are materializing the source data from the `driver_hourl The timestamps above should match the interval of data that has been computed by the data transformation system. -### 2.2. Automate periodic materializations +### 2.2. Automate periodic materializations It is up to you which orchestration/scheduler to use to periodically run `$ feast materialize`. Feast keeps the history of materialization in its registry so that the choice could be as simple as a [unix cron util](https://en.wikipedia.org/wiki/Cron). @@ -160,7 +160,7 @@ feature_refs = [ ] training_df = fs.get_historical_features( - entity_df=entity_df, + entity_df=entity_df, features=feature_refs, ).to_df() @@ -214,7 +214,7 @@ There are three approaches for that purpose sorted from the most simple one (in This approach is the most convenient to keep your infrastructure as minimalistic as possible and avoid deploying extra services. The Feast Python SDK will connect directly to the online store (Redis, Datastore, etc), pull the feature data, and run transformations locally (if required). -The obvious drawback is that your service must be written in Python to use the Feast Python SDK. +The obvious drawback is that your service must be written in Python to use the Feast Python SDK. A benefit of using a Python stack is that you can enjoy production-grade services with integrations with many existing data science tools. To integrate online retrieval into your service use the following code: @@ -245,9 +245,9 @@ This service will provide an HTTP API with JSON I/O, which can be easily used wi ### 4.3. Java based Feature Server deployed on Kubernetes For users with very latency-sensitive and high QPS use-cases, Feast offers a high-performance Java feature server. -Besides the benefits of running on JVM, this implementation also provides a gRPC API, which guarantees good connection utilization and -small request / response body size (compared to JSON). -You will need the Feast Java SDK to retrieve features from this service. This SDK wraps all the gRPC logic for you and provides more convenient APIs. +Besides the benefits of running on JVM, this implementation also provides a gRPC API, which guarantees good connection utilization and +small request / response body size (compared to JSON). +You will need the Feast Java SDK to retrieve features from this service. This SDK wraps all the gRPC logic for you and provides more convenient APIs. The Java based feature server can be deployed to Kubernetes cluster via Helm charts in a few simple steps: @@ -292,9 +292,9 @@ def feast_writer(spark_df): streamingDF.writeStream.foreachBatch(feast_writer).start() ``` -### 5.2. Push service *(still under development)* +### 5.2. Push Service (Alpha) -Alternatively, if you want to ingest features directly from a broker (eg, Kafka or Kinesis), you can use the "push service", which will write to an online store. +Alternatively, if you want to ingest features directly from a broker (eg, Kafka or Kinesis), you can use the "push service", which will write to an online store and/or offline store. This service will expose an HTTP API or when deployed on Serverless platforms like AWS Lambda or Google Cloud Run, this service can be directly connected to Kinesis or PubSub. @@ -310,6 +310,30 @@ We use an [InfluxDB-style extension](https://github.com/prometheus/statsd_export We chose StatsD since it's a de-facto standard with various implementations (eg, [1](https://github.com/prometheus/statsd_exporter), [2](https://github.com/influxdata/telegraf/blob/master/plugins/inputs/statsd/README.md)) and metrics can be easily exported to Prometheus, InfluxDB, AWS CloudWatch, etc. +## 7. Using environment variables in your yaml configuration + +You might want to dynamically set parts of your configuration from your environment. For instance to deploy Feast to production and development with the same configuration, but a different server. Or to inject secrets without exposing them in your git repo. To do this, it is possible to use the `${ENV_VAR}` syntax in your `feature_store.yaml` file. For instance: + +```yaml +project: my_project +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: ${REDIS_CONNECTION_STRING} +``` + +It is possible to set a default value if the environment variable is not set, with `${ENV_VAR:"default"}`. For instance: + +```yaml +project: my_project +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: ${REDIS_CONNECTION_STRING:"0.0.0.0:6379"} +``` + --- ## Summary diff --git a/docs/reference/data-sources/README.md b/docs/reference/data-sources/README.md index 89656b25702..b4fbc98b46f 100644 --- a/docs/reference/data-sources/README.md +++ b/docs/reference/data-sources/README.md @@ -22,6 +22,14 @@ Please see [Data Source](../../getting-started/concepts/feature-view.md#data-sou [push.md](push.md) {% endcontent-ref %} +{% content-ref url="kafka.md" %} +[kafka.md](kafka.md) +{% endcontent-ref %} + +{% content-ref url="kinesis.md" %} +[kinesis.md](kinesis.md) +{% endcontent-ref %} + {% content-ref url="spark.md" %} [spark.md](spark.md) {% endcontent-ref %} diff --git a/docs/reference/data-sources/kafka.md b/docs/reference/data-sources/kafka.md new file mode 100644 index 00000000000..8794c7a1e81 --- /dev/null +++ b/docs/reference/data-sources/kafka.md @@ -0,0 +1,75 @@ +# Kafka source + +**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. + +## Description + +Kafka sources allow users to register Kafka streams as data sources. Feast currently does not launch or monitor jobs to ingest data from Kafka. Users are responsible for launching and monitoring their own ingestion jobs, which should write feature values to the online store through [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). An example of how to launch such a job with Spark can be found [here](https://github.com/feast-dev/feast/tree/master/sdk/python/feast/infra/contrib). Feast also provides functionality to write to the offline store using the `write_to_offline_store` functionality. + +Kafka sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for writing data from their Kafka streams to a batch data source such as a data warehouse table. When using a Kafka source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. + +## Stream sources +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training (optional) +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance + +## Example +### Defining a Kafka source +Note that the Kafka source has a batch source. +```python +from datetime import timedelta + +from feast import Field, FileSource, KafkaSource, stream_feature_view +from feast.data_format import JsonFormat +from feast.types import Float32 + +driver_stats_batch_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", +) + +driver_stats_stream_source = KafkaSource( + name="driver_stats_stream", + kafka_bootstrap_servers="localhost:9092", + topic="drivers", + timestamp_field="event_timestamp", + batch_source=driver_stats_batch_source, + message_format=JsonFormat( + schema_json="driver_id integer, event_timestamp timestamp, conv_rate double, acc_rate double, created timestamp" + ), + watermark_delay_threshold=timedelta(minutes=5), +) +``` + +### Using the Kafka source in a stream feature view +The Kafka source can be used in a stream feature view. +```python +@stream_feature_view( + entities=[driver], + ttl=timedelta(seconds=8640000000), + mode="spark", + schema=[ + Field(name="conv_percentage", dtype=Float32), + Field(name="acc_percentage", dtype=Float32), + ], + timestamp_field="event_timestamp", + online=True, + source=driver_stats_stream_source, +) +def driver_hourly_stats_stream(df: DataFrame): + from pyspark.sql.functions import col + + return ( + df.withColumn("conv_percentage", col("conv_rate") * 100.0) + .withColumn("acc_percentage", col("acc_rate") * 100.0) + .drop("conv_rate", "acc_rate") + ) +``` + +### Ingesting data +See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. diff --git a/docs/reference/data-sources/kinesis.md b/docs/reference/data-sources/kinesis.md new file mode 100644 index 00000000000..f2adadfec03 --- /dev/null +++ b/docs/reference/data-sources/kinesis.md @@ -0,0 +1,74 @@ +# Kinesis source + +**Warning**: This is an _experimental_ feature. It's intended for early testing and feedback, and could change without warnings in future releases. + +## Description + +Kinesis sources allow users to register Kinesis streams as data sources. Feast currently does not launch or monitor jobs to ingest data from Kinesis. Users are responsible for launching and monitoring their own ingestion jobs, which should write feature values to the online store through [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). An example of how to launch such a job with Spark to ingest from Kafka can be found [here](https://github.com/feast-dev/feast/tree/master/sdk/python/feast/infra/contrib); by using a different plugin, the example can be adapted to Kinesis. Feast also provides functionality to write to the offline store using the `write_to_offline_store` functionality. + +Kinesis sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for writing data from their Kinesis streams to a batch data source such as a data warehouse table. When using a Kinesis source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. + +## Stream sources +Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: + +1. Raw events come in (stream 1) +2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) +3. Write stream 2 values to an offline store as a historical log for training (optional) +4. Write stream 2 values to an online store for low latency feature serving +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance + +## Example +### Defining a Kinesis source +Note that the Kinesis source has a batch source. +```python +from datetime import timedelta + +from feast import Field, FileSource, KinesisSource, stream_feature_view +from feast.data_format import JsonFormat +from feast.types import Float32 + +driver_stats_batch_source = FileSource( + name="driver_stats_source", + path="data/driver_stats.parquet", + timestamp_field="event_timestamp", +) + +driver_stats_stream_source = KinesisSource( + name="driver_stats_stream", + stream_name="drivers", + timestamp_field="event_timestamp", + batch_source=driver_stats_batch_source, + record_format=JsonFormat( + schema_json="driver_id integer, event_timestamp timestamp, conv_rate double, acc_rate double, created timestamp" + ), + watermark_delay_threshold=timedelta(minutes=5), +) +``` + +### Using the Kinesis source in a stream feature view +The Kinesis source can be used in a stream feature view. +```python +@stream_feature_view( + entities=[driver], + ttl=timedelta(seconds=8640000000), + mode="spark", + schema=[ + Field(name="conv_percentage", dtype=Float32), + Field(name="acc_percentage", dtype=Float32), + ], + timestamp_field="event_timestamp", + online=True, + source=driver_stats_stream_source, +) +def driver_hourly_stats_stream(df: DataFrame): + from pyspark.sql.functions import col + + return ( + df.withColumn("conv_percentage", col("conv_rate") * 100.0) + .withColumn("acc_percentage", col("acc_rate") * 100.0) + .drop("conv_rate", "acc_rate") + ) +``` + +### Ingesting data +See [here](https://github.com/feast-dev/streaming-tutorial) for a example of how to ingest data from a Kafka source into Feast. The approach used in the tutorial can be easily adapted to work for Kinesis as well. diff --git a/docs/reference/data-sources/push.md b/docs/reference/data-sources/push.md index e6eff312ec1..6af070d1c41 100644 --- a/docs/reference/data-sources/push.md +++ b/docs/reference/data-sources/push.md @@ -4,28 +4,27 @@ ## Description -Push sources allow feature values to be pushed to the online store in real time. This allows fresh feature values to be made available to applications. Push sources supercede the +Push sources allow feature values to be pushed to the online store and offline store in real time. This allows fresh feature values to be made available to applications. Push sources supercede the [FeatureStore.write_to_online_store](https://rtd.feast.dev/en/latest/index.html#feast.feature_store.FeatureStore.write_to_online_store). Push sources can be used by multiple feature views. When data is pushed to a push source, Feast propagates the feature values to all the consuming feature views. -Push sources must have a batch source specified, since that's the source used when retrieving historical features. -When using a PushSource as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the definition explicitly. +Push sources must have a batch source specified. The batch source will be used for retrieving historical features. Thus users are also responsible for pushing data to a batch data source such as a data warehouse table. When using a push source as a stream source in the definition of a feature view, a batch source doesn't need to be specified in the feature view definition explicitly. ## Stream sources Streaming data sources are important sources of feature values. A typical setup with streaming data looks like: 1. Raw events come in (stream 1) 2. Streaming transformations applied (e.g. generating features like `last_N_purchased_categories`) (stream 2) -3. Write stream 2 values to an offline store as a historical log for training +3. Write stream 2 values to an offline store as a historical log for training (optional) 4. Write stream 2 values to an online store for low latency feature serving -5. Periodically materialize feature values from the offline store into the online store for improved correctness +5. Periodically materialize feature values from the offline store into the online store for decreased training-serving skew and improved model performance -Feast now allows users to push features previously registered in a feature view to the online store for fresher features. +Feast allows users to push features previously registered in a feature view to the online store for fresher features. It also allows users to push batches of stream data to the offline store by specifying that the push be directed to the offline store. This will push the data to the offline store declared in the repository configuration used to initialize the feature store. ## Example ### Defining a push source -Note that the push schema needs to also include the entity +Note that the push schema needs to also include the entity. ```python from feast import PushSource, ValueType, BigQuerySource, FeatureView, Feature, Field @@ -45,14 +44,16 @@ fv = FeatureView( ``` ### Pushing data +Note that the `to` parameter is optional and defaults to online but we can specify these options: `PushMode.ONLINE`, `PushMode.OFFLINE`, or `PushMode.ONLINE_AND_OFFLINE`. ```python from feast import FeatureStore import pandas as pd +from feast.data_source import PushMode fs = FeatureStore(...) feature_data_frame = pd.DataFrame() -fs.push("push_source_name", feature_data_frame) +fs.push("push_source_name", feature_data_frame, to=PushMode.ONLINE_AND_OFFLINE) ``` -See also [Python feature server](../feature-servers/python-feature-server.md) for instructions on how to push data to a deployed feature server. +See also [Python feature server](../feature-servers/python-feature-server.md) for instructions on how to push data to a deployed feature server. diff --git a/docs/reference/feature-servers/go-feature-retrieval.md b/docs/reference/feature-servers/go-feature-retrieval.md index 30c1a9ca721..685e7201cb6 100644 --- a/docs/reference/feature-servers/go-feature-retrieval.md +++ b/docs/reference/feature-servers/go-feature-retrieval.md @@ -35,6 +35,36 @@ go_feature_retrieval: True ``` {% endcode %} +## Feature logging + +Go feature server can log all requested entities and served features to a configured destination inside an offline store. +This allows users to create new datasets from features served online. Those datasets could be used for future trainings or for +feature validations. To enable feature logging we need to edit `feature_store.yaml`: +```yaml +project: my_feature_repo +registry: data/registry.db +provider: local +online_store: + type: redis + connection_string: "localhost:6379" +go_feature_retrieval: True +feature_server: + feature_logging: + enable: True +``` + +Feature logging configuration in `feature_store.yaml` also allows to tweak some low-level parameters to achieve the best performance: +```yaml +feature_server: + feature_logging: + enable: True + flush_interval_secs: 300 + write_to_disk_interval_secs: 30 + emit_timeout_micro_secs: 10000 + queue_capacity: 10000 +``` +All these parameters are optional. + ## Future/Current Work The Go feature retrieval online feature logging for Data Quality Monitoring is currently in development. More information can be found [here](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit#heading=h.9gaqqtox9jg6). diff --git a/docs/reference/feature-servers/python-feature-server.md b/docs/reference/feature-servers/python-feature-server.md index 352f0edc167..ecc12dd12d6 100644 --- a/docs/reference/feature-servers/python-feature-server.md +++ b/docs/reference/feature-servers/python-feature-server.md @@ -2,7 +2,7 @@ ## Overview -The feature server is an HTTP endpoint that serves features with JSON I/O. This enables users to write + read features from Feast online stores using any programming language that can make HTTP requests. +The feature server is an HTTP endpoint that serves features with JSON I/O. This enables users to write + read features from Feast online stores using any programming language that can make HTTP requests. ## CLI @@ -152,9 +152,12 @@ curl -X POST \ }' | jq ``` -### Pushing features to the online store -You can push data corresponding to a push source to the online store (note that timestamps need to be strings): +### Pushing features to the online and offline stores +You can push data corresponding to a push source to the online and offline stores (note that timestamps need to be strings): +You can also define a pushmode to push stream or batch data, either to the online store, offline store, or both. The feature server will throw an error if the online/offline store doesn't support the push api functionality. + +The request definition for pushmode is a string parameter `to` where the options are: ["online", "offline", "online_and_offline"]. ```text curl -X POST "http://localhost:6566/push" -d '{ "push_source_name": "driver_hourly_stats_push_source", @@ -165,7 +168,8 @@ curl -X POST "http://localhost:6566/push" -d '{ "conv_rate": [1.0], "acc_rate": [1.0], "avg_daily_trips": [1000] - } + }, + "to": "online_and_offline", }' | jq ``` @@ -187,9 +191,10 @@ event_dict = { } push_data = { "push_source_name":"driver_stats_push_source", - "df":event_dict + "df":event_dict, + "to":"online", } requests.post( - "http://localhost:6566/push", + "http://localhost:6566/push", data=json.dumps(push_data)) ``` diff --git a/docs/roadmap.md b/docs/roadmap.md index c2f5511f1ed..19af4f95c91 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -4,7 +4,6 @@ The list below contains the functionality that contributors are planning to deve * Items below that are in development (or planned for development) will be indicated in parentheses. * We welcome contribution to all items in the roadmap! -* Want to influence our roadmap and prioritization? Submit your feedback to [this form](https://docs.google.com/forms/d/e/1FAIpQLSfa1nRQ0sKz-JEFnMMCi4Jseag\_yDssO\_3nV9qMfxfrkil-wA/viewform). * Want to speak to a Feast contributor? We are more than happy to jump on a call. Please schedule a time using [Calendly](https://calendly.com/d/x2ry-g5bb/meet-with-feast-team). * **Data Sources** @@ -37,17 +36,16 @@ The list below contains the functionality that contributors are planning to deve * [x] [Azure Cache for Redis (community plugin)](https://github.com/Azure/feast-azure) * [x] [Postgres (contrib plugin)](https://docs.feast.dev/reference/online-stores/postgres) * [x] [Custom online store support](https://docs.feast.dev/how-to-guides/adding-support-for-a-new-online-store) + * [x] [Cassandra / AstraDB](https://github.com/datastaxdevs/feast-cassandra-online-store) * [ ] Bigtable (in progress) - * [ ] Cassandra -* **Streaming** - * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) - * [x] [Push based streaming data ingestion](https://docs.feast.dev/reference/data-sources/push.md) - * [ ] Streaming ingestion on AWS - * [ ] Streaming ingestion on GCP * **Feature Engineering** * [x] On-demand Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1lgfIw0Drc65LpaxbUu49RCeJgMew547meSJttnUqz7c/edit#)) + * [x] Streaming Transformations (Alpha release. See [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit)) * [ ] Batch transformation (In progress. See [RFC](https://docs.google.com/document/d/1964OkzuBljifDvkV-0fakp2uaijnVzdwWNGdz7Vz50A/edit)) - * [ ] Streaming transformation +* **Streaming** + * [x] [Custom streaming ingestion job support](https://docs.feast.dev/how-to-guides/creating-a-custom-provider) + * [x] [Push based streaming data ingestion to online store (Alpha)](https://docs.feast.dev/reference/data-sources/push) + * [x] [Push based streaming data ingestion to offline store (Alpha)](https://docs.feast.dev/reference/data-sources/push) * **Deployments** * [x] AWS Lambda (Alpha release. See [RFC](https://docs.google.com/document/d/1eZWKWzfBif66LDN32IajpaG-j82LSHCCOzY6R7Ax7MI/edit)) * [x] Kubernetes (See [guide](https://docs.feast.dev/how-to-guides/running-feast-in-production#4.3.-java-based-feature-server-deployed-on-kubernetes)) @@ -61,7 +59,7 @@ The list below contains the functionality that contributors are planning to deve * [ ] Java Client * [ ] Go Client * [ ] Delete API - * [ ] Feature Logging (for training) + * [] Feature Logging (for training) * **Data Quality Management (See [RFC](https://docs.google.com/document/d/110F72d4NTv80p35wDSONxhhPBqWRwbZXG4f9mNEMd98/edit))** * [x] Data profiling and validation (Great Expectations) * [ ] Training-serving skew detection (in progress) @@ -72,6 +70,5 @@ The list below contains the functionality that contributors are planning to deve * [x] CLI for browsing feature registry * [x] Model-centric feature tracking (feature services) * [x] Amundsen integration (see [Feast extractor](https://github.com/amundsen-io/amundsen/blob/main/databuilder/databuilder/extractor/feast_extractor.py)) - * [x] Feast Web UI (Alpha release. See [documentation](https://docs.feast.dev/reference/alpha-web-ui.md)) + * [x] Feast Web UI (Alpha release. See [documentation](https://docs.feast.dev/reference/alpha-web-ui)) * [ ] REST API for browsing feature registry - * [ ] Feature versioning diff --git a/docs/tutorials/building-streaming-features.md b/docs/tutorials/building-streaming-features.md new file mode 100644 index 00000000000..01e02b81a93 --- /dev/null +++ b/docs/tutorials/building-streaming-features.md @@ -0,0 +1,5 @@ +# Building streaming features + +Feast supports registering streaming feature views and Kafka and Kinesis streaming sources. It also provides an interface for stream processing called the `Stream Processor`. An example Kafka/Spark StreamProcessor is implemented in the contrib folder. For more details, please see the [RFC](https://docs.google.com/document/d/1UzEyETHUaGpn0ap4G82DHluiCj7zEbrQLkJJkKSv4e8/edit?usp=sharing) for more details. + +Please see [here](https://github.com/feast-dev/streaming-tutorial) for a tutorial on how to build a versioned streaming pipeline that registers your transformations, features, and data sources in Feast. diff --git a/docs/tutorials/tutorials-overview.md b/docs/tutorials/tutorials-overview.md index 32e64071b06..9432783a690 100644 --- a/docs/tutorials/tutorials-overview.md +++ b/docs/tutorials/tutorials-overview.md @@ -11,3 +11,5 @@ These Feast tutorials showcase how to use Feast to simplify end to end model tra {% page-ref page="driver-stats-on-snowflake.md" %} {% page-ref page="validating-historical-features.md" %} + +{% page-ref page="using-scalable-registry.md" %} diff --git a/docs/tutorials/using-scalable-registry.md b/docs/tutorials/using-scalable-registry.md new file mode 100644 index 00000000000..51fa50ff337 --- /dev/null +++ b/docs/tutorials/using-scalable-registry.md @@ -0,0 +1,36 @@ +--- +description: >- + Tutorial on how to use the SQL registry for scalable registry updates +--- + +# Using Scalable Registry + +## Overview + +By default, the registry Feast uses a file-based registry implementation, which stores the protobuf representation of the registry as a serialized file. This registry file can be stored in a local file system, or in cloud storage (in, say, S3 or GCS). + +However, there's inherent limitations with a file-based registry, since changing a single field in the registry requires re-writing the whole registry file. With multiple concurrent writers, this presents a risk of data loss, or bottlenecks writes to the registry since all changes have to be serialized (e.g. when running materialization for multiple feature views or time ranges concurrently). + +An alternative to the file-based registry is the [SQLRegistry](https://rtd.feast.dev/en/latest/feast.infra.registry_stores.html#feast.infra.registry_stores.sql.SqlRegistry) which ships with Feast. This implementation stores the registry in a relational database, and allows for changes to individual objects atomically. +Under the hood, the SQL Registry implementation uses [SQLAlchemy](https://docs.sqlalchemy.org/en/14/) to abstract over the different databases. Consequently, any [database supported](https://docs.sqlalchemy.org/en/14/core/engines.html#supported-databases) by SQLAlchemy can be used by the SQL Registry. +Feast can use the SQL Registry via a config change in the feature_store.yaml file. An example of how to configure this would be: + +```yaml +project: +provider: +online_store: redis +offline_store: file +registry: + registry_type: sql + path: postgresql://postgres:mysecretpassword@127.0.0.1:55001/feast +``` + +Specifically, the registry_type needs to be set to sql in the registry config block. On doing so, the path should refer to the [Database URL](https://docs.sqlalchemy.org/en/14/core/engines.html#database-urls) for the database to be used, as expected by SQLAlchemy. No other additional commands are currently needed to configure this registry. + +There are some things to note about how the SQL registry works: +- Once instantiated, the Registry ensures the tables needed to store data exist, and creates them if they do not. +- Upon tearing down the feast project, the registry ensures that the tables are dropped from the database. +- The schema for how data is laid out in tables can be found . It is intentionally simple, storing the serialized protobuf versions of each Feast object keyed by its name. + +## Example Usage: Concurrent materialization +The SQL Registry should be used when materializing feature views concurrently to ensure correctness of data in the registry. This can be achieved by simply running feast materialize or feature_store.materialize multiple times using a correctly configured feature_store.yaml. This will make each materialization process talk to the registry database concurrently, and ensure the metadata updates are serialized. \ No newline at end of file diff --git a/examples/java-demo/feature_repo/driver_repo.py b/examples/java-demo/feature_repo/driver_repo.py index c91e5a40bed..e17a5d9cf89 100644 --- a/examples/java-demo/feature_repo/driver_repo.py +++ b/examples/java-demo/feature_repo/driver_repo.py @@ -7,14 +7,14 @@ from google.protobuf.duration_pb2 import Duration from feast.field import Field -from feast import Entity, Feature, BatchFeatureView, FileSource, ValueType +from feast import Entity, Feature, BatchFeatureView, FileSource driver_hourly_stats = FileSource( path="data/driver_stats_with_string.parquet", timestamp_field="event_timestamp", created_timestamp_column="created", ) -driver = Entity(name="driver_id", value_type=ValueType.INT64, description="driver id",) +driver = Entity(name="driver_id", description="driver id",) driver_hourly_stats_view = BatchFeatureView( name="driver_hourly_stats", entities=["driver_id"], diff --git a/examples/quickstart/quickstart.ipynb b/examples/quickstart/quickstart.ipynb index 60974d27513..d29ee4fa35f 100644 --- a/examples/quickstart/quickstart.ipynb +++ b/examples/quickstart/quickstart.ipynb @@ -1,20 +1,4 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "name": "Feast Codelab", - "provenance": [], - "collapsed_sections": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "markdown", @@ -54,28 +38,20 @@ }, { "cell_type": "code", + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "rXNMAAJKQPG5", - "outputId": "52297709-380b-4200-8e7c-3d0102a82ea4" + "outputId": "94fb2260-4453-45c9-ba77-5b384823a621" }, + "outputs": [], "source": [ "%%sh\n", "pip install feast -U -q\n", "pip install Pygments -q\n", "echo \"Please restart your runtime now (Runtime -> Restart runtime). This ensures that the correct dependencies are loaded.\"" - ], - "execution_count": 1, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Please restart your runtime now (Runtime -> Restart runtime). This ensures that the correct dependencies are loaded.\n" - ] - } ] }, { @@ -107,28 +83,32 @@ }, { "cell_type": "code", + "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IhirSkgUvYau", - "outputId": "df90af1a-06bd-48a1-94e6-7def19e87d5f" + "outputId": "664367b9-6a2a-493d-fd78-6495fb459fa2" }, - "source": [ - "!feast init feature_repo" - ], - "execution_count": 1, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Feast is an open source project that collects anonymized error reporting and usage statistics. To opt out or learn more see https://docs.feast.dev/reference/usage\n", + "/usr/local/lib/python3.7/dist-packages/scipy/fft/__init__.py:97: DeprecationWarning: The module numpy.dual is deprecated. Instead of using dual, use the functions directly from numpy or scipy.\n", + " from numpy.dual import register_func\n", + "/usr/local/lib/python3.7/dist-packages/scipy/sparse/sputils.py:17: DeprecationWarning: `np.typeDict` is a deprecated alias for `np.sctypeDict`.\n", + " supported_dtypes = [np.typeDict[x] for x in supported_dtypes]\n", "\n", "Creating a new Feast repository in \u001b[1m\u001b[32m/content/feature_repo\u001b[0m.\n", "\n" ] } + ], + "source": [ + "!feast init feature_repo" ] }, { @@ -150,22 +130,18 @@ }, { "cell_type": "code", + "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "9jXuzt4ovzA3", - "outputId": "bff15f0c-9f8e-4a3c-e605-5ad84be30709" + "outputId": "9e326892-f0cc-4d86-d0b2-f33f822f83a9" }, - "source": [ - "%cd feature_repo\n", - "!ls -R" - ], - "execution_count": 2, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "/content/feature_repo\n", ".:\n", @@ -175,6 +151,10 @@ "driver_stats.parquet\n" ] } + ], + "source": [ + "%cd feature_repo\n", + "!ls -R" ] }, { @@ -198,21 +178,18 @@ }, { "cell_type": "code", + "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "9_YJ--uYdtcP", - "outputId": "89268e31-6be0-43fb-e576-6d335a2c1dd9" + "outputId": "af56a8da-9ca2-4dd9-f73c-a60dd3e1613a" }, - "source": [ - "!pygmentize feature_store.yaml" - ], - "execution_count": 3, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "\u001b[94mproject\u001b[39;49;00m: feature_repo\n", "\u001b[94mregistry\u001b[39;49;00m: data/registry.db\n", @@ -221,6 +198,9 @@ " \u001b[94mpath\u001b[39;49;00m: data/online_store.db\n" ] } + ], + "source": [ + "!pygmentize feature_store.yaml" ] }, { @@ -236,56 +216,21 @@ }, { "cell_type": "code", + "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/", - "height": 424 + "height": 423 }, "id": "sIF2lO59dwzi", - "outputId": "80e798d5-df21-4ebd-de1c-9bde282bd742" + "outputId": "8931930b-b32f-43e1-d45b-de230489c7b8" }, - "source": [ - "import pandas as pd\n", - "\n", - "pd.read_parquet(\"data/driver_stats.parquet\")" - ], - "execution_count": 4, "outputs": [ { - "output_type": "execute_result", "data": { - "text/plain": [ - " event_timestamp driver_id conv_rate acc_rate \\\n", - "0 2022-03-31 14:00:00+00:00 1005 0.313336 0.231481 \n", - "1 2022-03-31 15:00:00+00:00 1005 0.959499 0.942614 \n", - "2 2022-03-31 16:00:00+00:00 1005 0.231786 0.313516 \n", - "3 2022-03-31 17:00:00+00:00 1005 0.886911 0.531613 \n", - "4 2022-03-31 18:00:00+00:00 1005 0.574945 0.718223 \n", - "... ... ... ... ... \n", - "1802 2022-04-15 12:00:00+00:00 1001 0.521622 0.266667 \n", - "1803 2022-04-15 13:00:00+00:00 1001 0.003188 0.535501 \n", - "1804 2021-04-12 07:00:00+00:00 1001 0.709081 0.823138 \n", - "1805 2022-04-08 02:00:00+00:00 1003 0.033297 0.053268 \n", - "1806 2022-04-08 02:00:00+00:00 1003 0.033297 0.053268 \n", - "\n", - " avg_daily_trips created \n", - "0 303 2022-04-15 14:34:10.056 \n", - "1 842 2022-04-15 14:34:10.056 \n", - "2 782 2022-04-15 14:34:10.056 \n", - "3 634 2022-04-15 14:34:10.056 \n", - "4 441 2022-04-15 14:34:10.056 \n", - "... ... ... \n", - "1802 406 2022-04-15 14:34:10.056 \n", - "1803 593 2022-04-15 14:34:10.056 \n", - "1804 997 2022-04-15 14:34:10.056 \n", - "1805 534 2022-04-15 14:34:10.056 \n", - "1806 534 2022-04-15 14:34:10.056 \n", - "\n", - "[1807 rows x 6 columns]" - ], "text/html": [ "\n", - "
\n", + "
\n", "
\n", "
\n", "