From 9aacb2e978c7c934ea6d67f2f039ecd036ff4691 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 15:20:56 -0700 Subject: [PATCH 1/4] Remove 3.7 wheels Signed-off-by: Kevin Zhang --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 1b0059f1a3..da84fc5e55 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -66,7 +66,7 @@ jobs: uses: pypa/cibuildwheel@v2.7.0 env: CIBW_BUILD: "cp3*_x86_64" - CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" + CIBW_SKIP: "cp36-* cp37-* *-musllinux_x86_64 cp310-macosx_x86_64" CIBW_ARCHS: "native" CIBW_ENVIRONMENT: > COMPILE_GO=True PATH=$PATH:/usr/local/go/bin @@ -150,7 +150,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-10.15 ] - python-version: [ "3.7", "3.8", "3.9", "3.10"] + python-version: [ "3.8", "3.9", "3.10"] from-source: [ True, False ] env: # this script is for testing servers From b555db8c232983fb3a962ae0ce3ab1153898283a Mon Sep 17 00:00:00 2001 From: Achal Shah Date: Mon, 9 May 2022 14:01:56 -0700 Subject: [PATCH 2/4] ci: Add a nightly CI job for integration tests Signed-off-by: Achal Shah --- .github/workflows/nightly-ci.yml | 114 +++++++++++++++++++++ .github/workflows/pr_integration_tests.yml | 6 +- 2 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/nightly-ci.yml diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml new file mode 100644 index 0000000000..fe9f686c2d --- /dev/null +++ b/.github/workflows/nightly-ci.yml @@ -0,0 +1,114 @@ +name: nightly-ci + +on: + schedule: + - cron: '00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST. + +# concurrency is currently broken, see details https://github.com/actions/runner/issues/1532 +#concurrency: +# group: pr-integration-tests-${{ github.event.pull_request.number }} +# cancel-in-progress: true + +jobs: + check_date: + runs-on: ubuntu-latest + name: Check latest commit + outputs: + WAS_EDITED: ${{ steps.check_date.outputs.WAS_EDITED }} + steps: + - uses: actions/checkout@v2 + with: + ref: develop + - id: check_date + name: Check if there were commits in the last day + if: ${{ github.event_name == 'schedule' }} + run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') + + integration-test-python: + # all jobs MUST have this if check for 'ok-to-test' or 'approved' or 'lgtm' for security purposes. + needs: [check_date] + runs-on: ${{ env.OS }} + env: + OS: "ubuntu-latest" + PYTHON: "3.7" + services: + redis: + image: redis + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v2 + with: + # pull_request_target runs the workflow in the context of the base repo + # as such actions/checkout needs to be explicit configured to retrieve + # code from the PR. + ref: refs/pull/${{ github.event.pull_request.number }}/merge + submodules: recursive + - name: Setup Python + uses: actions/setup-python@v2 + id: setup-python + with: + python-version: ${{ env.PYTHON }} + architecture: x64 + - name: Setup Go + id: setup-go + uses: actions/setup-go@v2 + with: + go-version: 1.17.7 + - name: Set up gcloud SDK + uses: google-github-actions/setup-gcloud@v0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_SA_KEY }} + export_default_credentials: true + - name: Use gcloud CLI + run: gcloud info + - name: Set up AWS SDK + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + - name: Use AWS CLI + run: aws sts get-caller-identity + - name: Upgrade pip version + run: | + pip install --upgrade "pip>=21.3.1" + - name: Get pip cache dir + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + - name: pip cache + uses: actions/cache@v2 + with: + path: | + ${{ steps.pip-cache.outputs.dir }} + /opt/hostedtoolcache/Python + /Users/runner/hostedtoolcache/Python + key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }} + restore-keys: | + ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- + - name: Install pip-tools + run: pip install pip-tools + - name: Install dependencies + run: make install-python-ci-dependencies + - name: Setup Redis Cluster + run: | + docker pull vishnunair/docker-redis-cluster:latest + docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster + - name: Test python + if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak + env: + FEAST_USAGE: "False" + IS_TEST: "True" + SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} + SNOWFLAKE_CI_USER: ${{ secrets.SNOWFLAKE_CI_USER }} + SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} + SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} + SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} + run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index db9e48fc2d..453586ccc4 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -87,8 +87,8 @@ jobs: python-version: [ "3.8" ] os: [ ubuntu-latest ] env: - OS: ${{ matrix.os }} - PYTHON: ${{ matrix.python-version }} + OS: "ubuntu-latest" + PYTHON: "3.7" services: redis: image: redis @@ -111,7 +111,7 @@ jobs: uses: actions/setup-python@v2 id: setup-python with: - python-version: ${{ matrix.python-version }} + python-version: ${{ os.PYTHON }} architecture: x64 - name: Setup Go id: setup-go From 04a362e3c361c0f04db581acdef674229ed61f15 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 15:48:46 -0700 Subject: [PATCH 3/4] Update and rebase Signed-off-by: Kevin Zhang --- .github/workflows/nightly-ci.yml | 44 +++++++++++++++++----- .github/workflows/pr_integration_tests.yml | 4 +- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index fe9f686c2d..06571c2dd3 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -25,12 +25,16 @@ jobs: run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false') integration-test-python: - # all jobs MUST have this if check for 'ok-to-test' or 'approved' or 'lgtm' for security purposes. needs: [check_date] - runs-on: ${{ env.OS }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [ "3.8" ] + os: [ ubuntu-latest ] env: - OS: "ubuntu-latest" - PYTHON: "3.7" + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} services: redis: image: redis @@ -41,7 +45,7 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 5 - steps: + steps: - uses: actions/checkout@v2 with: # pull_request_target runs the workflow in the context of the base repo @@ -53,13 +57,13 @@ jobs: uses: actions/setup-python@v2 id: setup-python with: - python-version: ${{ env.PYTHON }} + python-version: ${{ os.PYTHON }} architecture: x64 - name: Setup Go id: setup-go uses: actions/setup-go@v2 with: - go-version: 1.17.7 + go-version: 1.18.0 - name: Set up gcloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -78,7 +82,7 @@ jobs: run: aws sts get-caller-identity - name: Upgrade pip version run: | - pip install --upgrade "pip>=21.3.1" + pip install --upgrade "pip>=21.3.1,<22.1" - name: Get pip cache dir id: pip-cache run: | @@ -95,6 +99,18 @@ jobs: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip- - name: Install pip-tools run: pip install pip-tools + - name: Install apache-arrow on ubuntu + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt update + sudo apt install -y -V ca-certificates lsb-release wget + wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb + sudo apt update + sudo apt install -y -V libarrow-dev + - name: Install apache-arrow on macos + if: matrix.os == 'macOS-latest' + run: brew install apache-arrow - name: Install dependencies run: make install-python-ci-dependencies - name: Setup Redis Cluster @@ -104,6 +120,7 @@ jobs: - name: Test python if: ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak env: + FEAST_SERVER_DOCKER_IMAGE_TAG: ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }} FEAST_USAGE: "False" IS_TEST: "True" SNOWFLAKE_CI_DEPLOYMENT: ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }} @@ -111,4 +128,13 @@ jobs: SNOWFLAKE_CI_PASSWORD: ${{ secrets.SNOWFLAKE_CI_PASSWORD }} SNOWFLAKE_CI_ROLE: ${{ secrets.SNOWFLAKE_CI_ROLE }} SNOWFLAKE_CI_WAREHOUSE: ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }} - run: pytest -n 8 --cov=./ --cov-report=xml --verbose --color=yes sdk/python/tests --integration --durations=5 + run: pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: integrationtests + env_vars: OS,PYTHON + fail_ci_if_error: true + verbose: true \ No newline at end of file diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index 453586ccc4..d96464cc72 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -87,8 +87,8 @@ jobs: python-version: [ "3.8" ] os: [ ubuntu-latest ] env: - OS: "ubuntu-latest" - PYTHON: "3.7" + OS: ${{ matrix.os }} + PYTHON: ${{ matrix.python-version }} services: redis: image: redis From 9b5027a16dc2ae413ed1d04ae8df3368726d6526 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 11 Jul 2022 16:56:56 -0700 Subject: [PATCH 4/4] Fix Signed-off-by: Kevin Zhang --- .github/workflows/pr_integration_tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_integration_tests.yml b/.github/workflows/pr_integration_tests.yml index d96464cc72..8f64950a30 100644 --- a/.github/workflows/pr_integration_tests.yml +++ b/.github/workflows/pr_integration_tests.yml @@ -111,7 +111,7 @@ jobs: uses: actions/setup-python@v2 id: setup-python with: - python-version: ${{ os.PYTHON }} + python-version: ${{ matrix.python-version }} architecture: x64 - name: Setup Go id: setup-go @@ -191,4 +191,4 @@ jobs: flags: integrationtests env_vars: OS,PYTHON fail_ci_if_error: true - verbose: true + verbose: true \ No newline at end of file