From b889b36f0dccb8addb1cd49acc7075d6635d0e10 Mon Sep 17 00:00:00 2001
From: Danny Chiao <danny@tecton.ai>
Date: Tue, 9 Aug 2022 20:13:20 -0400
Subject: [PATCH 1/3] ci: In nightly CI, cleanup dynamo tables related to
 integration tests

Signed-off-by: Danny Chiao <danny@tecton.ai>
---
 infra/scripts/cleanup_dynamo_ci.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)
 create mode 100644 infra/scripts/cleanup_dynamo_ci.py

diff --git a/infra/scripts/cleanup_dynamo_ci.py b/infra/scripts/cleanup_dynamo_ci.py
new file mode 100644
index 00000000000..997f5111f0b
--- /dev/null
+++ b/infra/scripts/cleanup_dynamo_ci.py
@@ -0,0 +1,16 @@
+import boto3
+
+
+def main() -> None:
+    db = boto3.resource("dynamodb")
+
+    num_deleted = 0
+    for table in db.tables.all():
+        if "integration_test" in table.name:
+            table.delete()
+            num_deleted += 1
+    print(f"Deleted {num_deleted} CI DynamoDB tables")
+
+
+if __name__ == "__main__":
+    main()

From 3993023df8637866cabcaf33ce378c901405652c Mon Sep 17 00:00:00 2001
From: Danny Chiao <danny@tecton.ai>
Date: Tue, 9 Aug 2022 20:14:26 -0400
Subject: [PATCH 2/3] workflow

Signed-off-by: Danny Chiao <danny@tecton.ai>
---
 .github/workflows/nightly-ci.yml | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index 0c2ba6a66a5..fc7542e81f1 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -24,6 +24,30 @@ jobs:
         name: Check if there were commits in the last day
         if: ${{ github.event_name == 'schedule' }}
         run: echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
+  cleanup_dynamo_tables:
+    if: github.repository == 'feast-dev/feast'
+    runs-on: ubuntu-latest
+    name: Cleanup dynamo tables which can fail to cleanup
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        id: setup-python
+        with:
+          python-version: "3.8"
+          architecture: x64
+      - name: Set up AWS SDK
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-west-2
+      - name: Install boto3
+        run: pip install boto3
+      - name: Run DynamoDB cleanup script
+        run: python infra/scripts/cleanup_dynamo_ci.py
   build-docker-image:
     if: github.repository == 'feast-dev/feast'
     needs: [check_date]
@@ -82,7 +106,7 @@ jobs:
       DOCKER_IMAGE_TAG: ${{ steps.image-tag.outputs.DOCKER_IMAGE_TAG }}
   integration-test-python:
     if: github.repository == 'feast-dev/feast'
-    needs: [check_date, build-docker-image]
+    needs: [check_date, build-docker-image, cleanup_dynamo_tables]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

From 4d858d6464665bc01528b86943bc4b9213f2aac2 Mon Sep 17 00:00:00 2001
From: Danny Chiao <danny@tecton.ai>
Date: Mon, 15 Aug 2022 10:35:29 -0400
Subject: [PATCH 3/3] add tqdm

Signed-off-by: Danny Chiao <danny@tecton.ai>
---
 infra/scripts/cleanup_dynamo_ci.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/infra/scripts/cleanup_dynamo_ci.py b/infra/scripts/cleanup_dynamo_ci.py
index 997f5111f0b..2dda36cc5a5 100644
--- a/infra/scripts/cleanup_dynamo_ci.py
+++ b/infra/scripts/cleanup_dynamo_ci.py
@@ -1,15 +1,21 @@
 import boto3
+from tqdm import tqdm
 
 
 def main() -> None:
     db = boto3.resource("dynamodb")
 
-    num_deleted = 0
-    for table in db.tables.all():
+    num_to_delete = 0
+    all_tables = db.tables.all()
+    for table in all_tables:
         if "integration_test" in table.name:
-            table.delete()
-            num_deleted += 1
-    print(f"Deleted {num_deleted} CI DynamoDB tables")
+            num_to_delete += 1
+    with tqdm(total=num_to_delete) as progress:
+        for table in all_tables:
+            if "integration_test" in table.name:
+                table.delete()
+                progress.update()
+    print(f"Deleted {num_to_delete} CI DynamoDB tables")
 
 
 if __name__ == "__main__":