fix(tests): Add Spark testsuite to tox.ini and to CI (getsentry#3199)

sentrivana · web-flow · commit bcc563cd7987 · 2024-06-25T13:51:58.000+02:00
diff --git a/.github/workflows/test-integrations-data-processing.yml b/.github/workflows/test-integrations-data-processing.yml
@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6","3.7","3.8","3.11","3.12"]
+        python-version: ["3.6","3.7","3.8","3.10","3.11","3.12"]
         # python3.6 reached EOL and is no longer being supported on
         # new versions of hosted runners on Github Actions
         # ubuntu-20.04 is the last version that supported python3.6
@@ -64,6 +64,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-rq-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+      - name: Test spark latest
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-spark-latest" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
       - name: Generate coverage XML
         run: |
           coverage combine .coverage*
@@ -118,6 +122,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-rq" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
+      - name: Test spark pinned
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-spark" --cov=tests --cov=sentry_sdk --cov-report= --cov-branch
       - name: Generate coverage XML
         run: |
           coverage combine .coverage*
diff --git a/scripts/split-tox-gh-actions/split-tox-gh-actions.py b/scripts/split-tox-gh-actions/split-tox-gh-actions.py
@@ -82,6 +82,7 @@
         "celery",
         "huey",
         "rq",
+        "spark",
     ],
     "Databases": [
         "asyncpg",
diff --git a/tox.ini b/tox.ini
@@ -6,7 +6,7 @@
 [tox]
 requires =
     # This version introduced using pip 24.1 which does not work with older Celery and HTTPX versions.
-    virtualenv<20.26.3  
+    virtualenv<20.26.3
 envlist =
     # === Common ===
     {py3.6,py3.7,py3.8,py3.9,py3.10,py3.11,py3.12}-common
@@ -219,6 +219,10 @@ envlist =
     {py3.7,py3.11}-sanic-v{23}
     {py3.8,py3.11}-sanic-latest
 
+    # Spark
+    {py3.8,py3.10,py3.11}-spark-v{3.1,3.3,3.5}
+    {py3.8,py3.10,py3.11}-spark-latest
+
     # Starlette
     {py3.7,py3.10}-starlette-v{0.19}
     {py3.7,py3.11}-starlette-v{0.20,0.24,0.28}
@@ -564,6 +568,12 @@ deps =
     sanic-v23: sanic~=23.0
     sanic-latest: sanic
 
+    # Spark
+    spark-v3.1: pyspark~=3.1.0
+    spark-v3.3: pyspark~=3.3.0
+    spark-v3.5: pyspark~=3.5.0
+    spark-latest: pyspark
+
     # Starlette
     starlette: pytest-asyncio
     starlette: python-multipart
@@ -643,6 +653,7 @@ setenv =
     gcp: TESTPATH=tests/integrations/gcp
     gql: TESTPATH=tests/integrations/gql
     graphene: TESTPATH=tests/integrations/graphene
+    grpc: TESTPATH=tests/integrations/grpc
     httpx: TESTPATH=tests/integrations/httpx
     huey: TESTPATH=tests/integrations/huey
     huggingface_hub: TESTPATH=tests/integrations/huggingface_hub
@@ -659,14 +670,14 @@ setenv =
     requests: TESTPATH=tests/integrations/requests
     rq: TESTPATH=tests/integrations/rq
     sanic: TESTPATH=tests/integrations/sanic
+    spark: TESTPATH=tests/integrations/spark
     starlette: TESTPATH=tests/integrations/starlette
     starlite: TESTPATH=tests/integrations/starlite
     sqlalchemy: TESTPATH=tests/integrations/sqlalchemy
     strawberry: TESTPATH=tests/integrations/strawberry
     tornado: TESTPATH=tests/integrations/tornado
     trytond: TESTPATH=tests/integrations/trytond
     socket: TESTPATH=tests/integrations/socket
-    grpc: TESTPATH=tests/integrations/grpc
 
     COVERAGE_FILE=.coverage-{envname}
 passenv =
