name: Flaky Tests

on:

push:

branches:

- main

- ci

workflow_dispatch:

env:

NODE_VERSION: 12.15.0

PYTHON_VERSION: 3.8

CACHE_NPM_DEPS: cache-npm

CACHE_OUT_DIRECTORY: cache-out-directory

CACHE_PIP_DEPS: cache-pip

CI_PYTHON_PATH: python

TEST_RESULTS_DIRECTORY: .

jobs:

tests:

name: Tests (with Python)

# The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.

runs-on: ${{ matrix.os }}

if: github.repository == 'microsoft/vscode-jupyter'

strategy:

fail-fast: false

matrix:

# We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,

# macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.

os: [ubuntu-latest, windows-latest]

python: [3.6, 3.7, 3.8]

test-suite: [python-unit, single-workspace, functional, functional-with-jupyter]

steps:

- name: Checkout

uses: actions/checkout@v2

- name: Use Python ${{matrix.python}}

uses: actions/setup-python@v2

with:

python-version: ${{matrix.python}}

- name: Upgrade pip

run: python -m pip install -U pip

- name: Use Node ${{env.NODE_VERSION}}

uses: actions/setup-node@v2.1.1

with:

node-version: ${{env.NODE_VERSION}}

# Start caching

# Cache Python Dependencies.

# Caching (https://github.com/actions/cache/blob/main/examples.md#python---pip

- name: Cache pip on linux

uses: actions/cache@v2

# if: startsWith(runner.os, 'Linux')

if: startsWith(matrix.test-suite, 'functional') && matrix.os == 'ubuntu-latest'

with:

path: ~/.cache/pip

key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}

restore-keys: |

${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-

- name: Cache pip on mac

uses: actions/cache@v2

if: startsWith(matrix.test-suite, 'functional') && matrix.os == 'macos-latest'

with:

path: ~/Library/Caches/pip

key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}

restore-keys: |

${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-

- name: Cache pip on windows

uses: actions/cache@v2

# if: startsWith(runner.os, 'Windows')

if: startsWith(matrix.test-suite, 'functional') && matrix.os == 'windows-latest'

with:

path: ~\AppData\Local\pip\Cache

key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{ hashFiles('requirements.txt') }}-${{hashFiles('build/debugger-install-requirements.txt')}}-${{hashFiles('test-requirements.txt')}}-${{hashFiles('ipython-test-requirements.txt')}}-${{hashFiles('functional-test-requirements.txt')}}-${{hashFiles('conda-functional-requirements.txt')}}

restore-keys: |

${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-

# Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)

- name: Cache npm on linux/mac

uses: actions/cache@v2

if: matrix.os != 'windows-latest'

with:

path: ~/.npm

key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

restore-keys: |

${{ runner.os }}-node-

- name: Get npm cache directory

if: matrix.os == 'windows-latest'

id: npm-cache

run: |

echo "::set-output name=dir::$(npm config get cache)"

- name: Cache npm on windows

uses: actions/cache@v2

if: matrix.os == 'windows-latest'

with:

path: ${{ steps.npm-cache.outputs.dir }}

key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}

restore-keys: |

${{ runner.os }}-node-

- name: Cache compiled TS files

# Use an id for this step so that its cache-hit output can be accessed and checked in the next step.

id: out-cache

uses: actions/cache@v2

with:

path: ./out

key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}

# For faster/better builds of sdists.

- run: python -m pip install wheel

if: startsWith(matrix.test-suite, 'functional')

shell: bash

# debugpy & Jedi LS is not shipped, only installed for local tests.

# In production, we get debugpy & LS from python extension.

- name: Install functional test requirements

# python -m pip --disable-pip-version-check install -t ./pythonFiles/lib/python --no-cache-dir --implementation py --no-deps --upgrade -r requirements.txt

run: |

python -m pip --disable-pip-version-check install -r build/debugger-install-requirements.txt

python ./pythonFiles/install_debugpy.py

python -m pip install numpy

python -m pip install --upgrade -r build/test-requirements.txt

python -m pip install --upgrade -r ./build/ipython-test-requirements.txt

python -m pip install --upgrade -r ./build/conda-functional-requirements.txt

- name: Install dependencies (npm ci)

run: npm ci --prefer-offline

- name: Compile if not cached

run: npx gulp prePublishNonBundle

if: steps.out-cache.outputs.cache-hit == false

# Run the Python and IPython tests in our codebase.

- name: Run Python and IPython unit tests

run: |

python pythonFiles/tests/run_all.py

python -m IPython pythonFiles/tests/run_all.py

if: matrix.test-suite == 'python-unit'

- name: Run single-workspace tests

env:

CI_PYTHON_VERSION: ${{matrix.python}}

uses: GabrielBB/xvfb-action@v1.4

with:

run: npm run testSingleWorkspace

if: matrix.test-suite == 'single-workspace'

- name: Run functional tests

run: npm run test:functional

if: matrix.test-suite == 'functional'

- name: Run functional tests with Jupyter

run: npm run test:functional

env:

VSCODE_PYTHON_ROLLING: 1

VSC_PYTHON_FORCE_LOGGING: 1

if: matrix.test-suite == 'functional-with-jupyter'