ci: In nightly CI, cleanup dynamo tables related to integration tests (feast-dev#3055)

adchia · web-flow · commit 41851be3db6d · 2022-08-15T10:12:36.000-07:00
* ci: In nightly CI, cleanup dynamo tables related to integration tests

Signed-off-by: Danny Chiao &lt;danny@tecton.ai&gt;

* workflow

Signed-off-by: Danny Chiao &lt;danny@tecton.ai&gt;

* add tqdm

Signed-off-by: Danny Chiao &lt;danny@tecton.ai&gt;

Signed-off-by: Danny Chiao &lt;danny@tecton.ai&gt;
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
@@ -24,6 +24,30 @@ jobs:
         name: Check if there were commits in the last day
         if: ${{ github.event_name == 'schedule' }}
         run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
+  cleanup_dynamo_tables:
+    if: github.repository == 'feast-dev/feast'
+    runs-on: ubuntu-latest
+    name: Cleanup dynamo tables which can fail to cleanup
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: "3.8"
+          architecture: x64
+      - name: Set up AWS SDK
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Install boto3
+        run: pip install boto3
+      - name: Run DynamoDB cleanup script
+        run: python infra/scripts/cleanup_dynamo_ci.py
   build-docker-image:
     if: github.repository == 'feast-dev/feast'
     needs: [check_date]
@@ -82,7 +106,7 @@ jobs:
       DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     if: github.repository == 'feast-dev/feast'
-    needs: [check_date, build-docker-image]
+    needs: [check_date, build-docker-image, cleanup_dynamo_tables]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
diff --git a/infra/scripts/cleanup_dynamo_ci.py b/infra/scripts/cleanup_dynamo_ci.py
@@ -0,0 +1,22 @@
+import boto3
+from tqdm import tqdm
+
+
+def main() -> None:
+    db = boto3.resource("dynamodb")
+
+    num_to_delete = 0
+    all_tables = db.tables.all()
+    for table in all_tables:
+        if "integration_test" in table.name:
+            num_to_delete += 1
+    with tqdm(total=num_to_delete) as progress:
+        for table in all_tables:
+            if "integration_test" in table.name:
+                table.delete()
+                progress.update()
+    print(f"Deleted {num_to_delete} CI DynamoDB tables")
+
+
+if __name__ == "__main__":
+    main()
