name: Tests

on:

workflow_dispatch:

push:

branches:

- main

- develop

tags:

- "v*.*.*"

pull_request:

branches:

- main

- develop

concurrency:

group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

cancel-in-progress: true

jobs:

test:

name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }})

runs-on: ${{ matrix.os }}

strategy:

matrix:

python-version: ["3.9"]

scikit-learn: ["1.0.*", "1.1.*", "1.2.*", "1.3.*", "1.4.*", "1.5.*"]

os: [ubuntu-latest]

sklearn-only: ["true"]

include:

- os: ubuntu-latest

python-version: "3.8" # no scikit-learn 0.23 release for Python 3.9

scikit-learn: "0.23.1"

sklearn-only: "true"

# scikit-learn 0.24 relies on scipy defaults, so we need to fix the version

# c.f. https://github.com/openml/openml-python/pull/1267

- os: ubuntu-latest

python-version: "3.9"

scikit-learn: "0.24"

scipy: "1.10.0"

sklearn-only: "true"

# Do a Windows and Ubuntu test for _all_ openml functionality

# I am not sure why these are on 3.8 and older scikit-learn

- os: windows-latest

python-version: "3.8"

scikit-learn: 0.24.*

scipy: "1.10.0"

sklearn-only: 'false'

# Include a code cov version

- os: ubuntu-latest

code-cov: true

python-version: "3.8"

scikit-learn: 0.23.1

sklearn-only: 'false'

fail-fast: false

steps:

- uses: actions/checkout@v4

with:

fetch-depth: 2

- name: Setup Python ${{ matrix.python-version }}

if: matrix.os != 'windows-latest' # windows-latest only uses preinstalled Python (3.9.13)

uses: actions/setup-python@v5

with:

python-version: ${{ matrix.python-version }}

- name: Install test dependencies

run: |

python -m pip install --upgrade pip

pip install -e .[test]

- name: Install scikit-learn ${{ matrix.scikit-learn }}

run: |

pip install scikit-learn==${{ matrix.scikit-learn }}

- name: Install numpy for Python 3.8

# Python 3.8 & scikit-learn<0.24 requires numpy<=1.23.5

if: ${{ matrix.python-version == '3.8' && matrix.scikit-learn == '0.23.1' }}

run: |

pip install numpy==1.23.5

- name: "Install NumPy 1.x and SciPy <1.11 for scikit-learn < 1.4"

if: ${{ contains(fromJSON('["1.0.*", "1.1.*", "1.2.*", "1.3.*"]'), matrix.scikit-learn) }}

run: |

# scipy has a change to the 'mode' behavior which breaks scikit-learn < 1.4

# numpy 2.0 has several breaking changes

pip install "numpy<2.0" "scipy<1.11"

- name: Install scipy ${{ matrix.scipy }}

if: ${{ matrix.scipy }}

run: |

pip install scipy==${{ matrix.scipy }}

- name: Store repository status

id: status-before

if: matrix.os != 'windows-latest'

run: |

git_status=$(git status --porcelain -b)

echo "BEFORE=$git_status" >> $GITHUB_ENV

echo "Repository status before tests: $git_status"

- name: Show installed dependencies

run: python -m pip list

- name: Run tests on Ubuntu Test

if: matrix.os == 'ubuntu-latest'

run: |

if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi

# Most of the time, running only the scikit-learn tests is sufficient

if [ ${{ matrix.sklearn-only }} = 'true' ]; then marks='sklearn and not production'; else marks='not production'; fi

echo pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

- name: Run tests on Ubuntu Production

if: matrix.os == 'ubuntu-latest'

run: |

if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi

# Most of the time, running only the scikit-learn tests is sufficient

if [ ${{ matrix.sklearn-only }} = 'true' ]; then marks='sklearn and production'; else marks='production'; fi

echo pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"

- name: Run tests on Windows

if: matrix.os == 'windows-latest'

run: | # we need a separate step because of the bash-specific if-statement in the previous one.

pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1

- name: Check for files left behind by test

if: matrix.os != 'windows-latest' && always()

run: |

before="${{ env.BEFORE }}"

after="$(git status --porcelain -b)"

if [[ "$before" != "$after" ]]; then

echo "git status from before: $before"

echo "git status from after: $after"

echo "Not all generated files have been deleted!"

exit 1

fi

- name: Upload coverage

if: matrix.code-cov && always()

uses: codecov/codecov-action@v4

with:

files: coverage.xml

token: ${{ secrets.CODECOV_TOKEN }}

fail_ci_if_error: true

verbose: true