name: CI

on:

workflow_call:

inputs:

config-path:

type: string

secrets:

JINA_AUTH_TOKEN:

required: false

pull_request:

types: [opened, synchronize, reopened]

jobs:

lint-ruff:

runs-on: ubuntu-latest

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python 3.8

uses: actions/setup-python@v4

with:

python-version: 3.8

- name: Lint with ruff

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install

# stop the build if there are Python syntax errors or undefined names

poetry run ruff docarray

# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide

poetry run ruff docarray

check-black:

runs-on: ubuntu-latest

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python 3.8

uses: actions/setup-python@v4

with:

python-version: 3.8

- name: check black

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --only dev

poetry run black --check .

env:

CHANGED_FILES: ${{ steps.file_changes.outputs.added_modified }}

import-test:

runs-on: ubuntu-latest

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python 3.8

uses: actions/setup-python@v4

with:

python-version: 3.8

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --without dev

poetry run pip install tensorflow==2.12.0

poetry run pip install jax

poetry run pip uninstall -y torch

poetry run pip install torch

- name: Test basic import

run: poetry run python -c 'from docarray import DocList, BaseDoc'

# it is time to say bye bye to mypy because of the way we handle support of pydantic v1 and v2

# check-mypy:

# runs-on: ubuntu-latest

# steps:

# - uses: actions/checkout@v2.5.0

# - name: Set up Python 3.8

# uses: actions/setup-python@v4

# with:

# python-version: 3.8

# - name: check mypy

# run: |

# python -m pip install --upgrade pip

# python -m pip install poetry

# poetry install --all-extras

# poetry run mypy docarray

docarray-test:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.9]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

test-path: [tests/integrations, tests/units, tests/documentation]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

poetry run pip install elasticsearch==8.6.2

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip uninstall -y torch

poetry run pip install torch

poetry run pip install numpy==1.26.1

sudo apt-get update

sudo apt-get install --no-install-recommends ffmpeg

- name: Test

id: test

run: |

poetry run pytest -m "not (tensorflow or benchmark or index or jax)" --cov=docarray --cov-report=xml -v -s ${{ matrix.test-path }} --ignore=tests/integrations/store/test_jac.py

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

env:

JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-test-proto3:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip install protobuf==3.20.0 # we check that we support 3.19

poetry run pip uninstall -y torch

poetry run pip install torch

sudo apt-get update

sudo apt-get install --no-install-recommends ffmpeg

- name: Test

id: test

run: |

poetry run pytest -m 'proto' --cov=docarray --cov-report=xml -v -s tests

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-doc-index:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

db_test_folder: [base_classes, elastic, epsilla, hnswlib, qdrant, weaviate, redis, milvus]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip install protobuf==3.20.0

poetry run pip install tensorflow==2.12.0

poetry run pip uninstall -y torch

poetry run pip install torch

sudo apt-get update

sudo apt-get install --no-install-recommends ffmpeg

- name: Test

id: test

run: |

poetry run pytest -m 'index and not elasticv8' --cov=docarray --cov-report=xml -v -s tests/index/${{ matrix.db_test_folder }}

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-elastic-v8:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip install protobuf==3.20.0

poetry run pip install tensorflow==2.12.0

poetry run pip install elasticsearch==8.6.2

poetry run pip uninstall -y torch

poetry run pip install torch

sudo apt-get update

sudo apt-get install --no-install-recommends ffmpeg

- name: Test

id: test

run: |

poetry run pytest -m 'index and elasticv8' --cov=docarray --cov-report=xml -v -s tests

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-test-tensorflow:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip install protobuf==3.20.0

poetry run pip install google-auth==2.23.0

poetry run pip install tensorflow==2.12.0

poetry run pip uninstall -y torch

poetry run pip install torch

sudo apt-get update

sudo apt-get install --no-install-recommends ffmpeg

- name: Test

id: test

run: |

poetry run pytest -m 'tensorflow' --cov=docarray --cov-report=xml -v -s tests

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-test-jax:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

pydantic-version: ["pydantic-v2", "pydantic-v1"]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

./scripts/install_pydantic_v2.sh ${{ matrix.pydantic-version }}

poetry run pip uninstall -y torch

poetry run pip install torch

poetry run pip install jaxlib

poetry run pip install jax

- name: Test

id: test

run: |

poetry run pytest -m 'jax' --cov=docarray --cov-report=xml -v -s tests

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

docarray-test-benchmarks:

needs: [import-test]

runs-on: ubuntu-latest

strategy:

fail-fast: false

matrix:

python-version: [3.8]

steps:

- uses: actions/checkout@v2.5.0

- name: Set up Python ${{ matrix.python-version }}

uses: actions/setup-python@v4

with:

python-version: ${{ matrix.python-version }}

- name: Prepare environment

run: |

python -m pip install --upgrade pip

python -m pip install poetry==1.7.1

poetry install --all-extras

poetry run pip uninstall -y torch

poetry run pip install torch

- name: Test

id: test

run: |

poetry run pytest -m 'benchmark' --cov=docarray --cov-report=xml -v -s tests

echo "flag it as docarray for codeoverage"

echo "codecov_flag=docarray" >> $GITHUB_OUTPUT

timeout-minutes: 30

- name: Check codecov file

id: check_files

uses: andstor/file-existence-action@v1

with:

files: "coverage.xml"

- name: Upload coverage from test to Codecov

uses: codecov/codecov-action@v3.1.1

if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.8'

with:

file: coverage.xml

name: benchmark-test-codecov

flags: ${{ steps.test.outputs.codecov_flag }}

fail_ci_if_error: false

# just for blocking the merge until all parallel tests are successful

success-all-test:

needs: [docarray-test, docarray-test-proto3, docarray-doc-index, docarray-elastic-v8, docarray-test-tensorflow, docarray-test-benchmarks, import-test, check-black, lint-ruff]

if: always()

runs-on: ubuntu-latest

steps:

- uses: technote-space/workflow-conclusion-action@v2

- name: Check Failure

if: env.WORKFLOW_CONCLUSION == 'failure'

run: exit 1

- name: Success

if: ${{ success() }}

run: echo "All Done"