# Regression Detection Suite

#

# This workflow runs under the following conditions:

# - in the merge queue if any source files were modified, added or deleted.

# - on demand by a PR comment matching either of:

# - '/ci-run-regression'

# - '/ci-run-all'

# (the comment issuer must be a member of the Vector GH team)

#

# This workflow runs our regression detection experiments, which are relative

# evaluations of the base SHA and head SHA, whose determination depends on how

# the workflow is invoked.

#

# The goal is to give quick-ish feedback on all-up Vector for a variety of configs

# as to whether throughput performance has gone down, gotten more variable in the

# pushed SHA.

#

# Regression detection is always done relative to the pushed SHA, meaning any

# changes you introduce to the experiment will be picked up both for the base

# SHA variant and your current SHA.

#

# Docker image tags are SHA-SHA. The first SHA is the one that triggered this

# workflow, the second is the one of the Vector being tested.

# For comparison images the two SHAs are identical.

name: Regression Detection Suite

on:

merge_group:

types: [checks_requested]

workflow_call:

# Don't want to run this on each PR commit, but because GH doesn't allow specifying different required checks

# for pull request and merge queue, we need to "run" it in pull request, but in the jobs we will just auto pass.

pull_request:

concurrency:

group: ${{ github.workflow }}-${{ github.event.comment.html_url || github.event.merge_group.head_sha || github.sha }}

cancel-in-progress: true

env:

SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }}

jobs:

# Only run this workflow if files changed in areas that could possibly introduce a regression

should-run:

runs-on: ubuntu-latest

if: github.event_name != 'pull_request'

outputs:

source_changed: ${{ steps.filter.outputs.SOURCE_CHANGED }}

comment_valid: ${{ steps.comment.outputs.isTeamMember }}

steps:

- uses: actions/checkout@v3

- name: Collect file changes

id: changes

if: github.event_name == 'merge_group'

uses: dorny/paths-filter@v2

with:

base: ${{ github.event.merge_group.base_ref }}

ref: ${{ github.event.merge_group.head_ref }}

list-files: shell

filters: |

all_changed:

- added|deleted|modified: "**"

ignore:

- "./.github/**"

- "./.gitignore"

- "distribution/**"

- "rust-doc/**"

- "docs/**"

- "rfcs/**"

- "testing/**"

- "tilt/**"

- "website/**"

- "*.md"

- "Tiltfile"

- "netlify.toml"

- "NOTICE"

- "LICENSE-3rdparty.csv"

- "LICENSE"

# This step allows us to conservatively run the tests if we added a new

# file or directory for source code, but forgot to add it to this workflow.

# Instead, we may unnecessarily run the test on new file or dir additions that

# wouldn't likely introduce regressions.

- name: Determine if should not run due to irrelevant file changes

id: filter

if: github.event_name == 'merge_group'

env:

ALL: ${{ steps.changes.outputs.all_changed_files }}

IGNORE: ${{ steps.changes.outputs.ignore_files }}

run: |

echo "ALL='${{ env.ALL }}'"

echo "IGNORE='${{ env.IGNORE }}'"

export SOURCE_CHANGED=$(comm -2 -3 <(printf "%s\n" "${{ env.ALL }}") <(printf "%s\n" "${{ env.IGNORE }}"))

echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"

if [ "${SOURCE_CHANGED}" == "" ]; then

export SOURCE_CHANGED="false"

else

export SOURCE_CHANGED="true"

fi

echo "SOURCE_CHANGED='${SOURCE_CHANGED}'"

echo "SOURCE_CHANGED=${SOURCE_CHANGED}" >> $GITHUB_OUTPUT

compute-metadata:

name: Compute metadata

runs-on: ubuntu-22.04

needs: should-run

if: github.event_name != 'merge_group' || needs.should-run.outputs.source_changed == 'true'

outputs:

pr-number: ${{ steps.pr-metadata-merge-queue.outputs.PR_NUMBER || steps.pr-metadata-comment.outputs.PR_NUMBER }}

baseline-sha: ${{ steps.pr-metadata-merge-queue.outputs.BASELINE_SHA || steps.pr-metadata-comment.outputs.BASELINE_SHA }}

baseline-tag: ${{ steps.pr-metadata-merge-queue.outputs.BASELINE_TAG || steps.pr-metadata-comment.outputs.BASELINE_TAG }}

comparison-sha: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_SHA || steps.pr-metadata-comment.outputs.COMPARISON_SHA }}

comparison-tag: ${{ steps.pr-metadata-merge-queue.outputs.COMPARISON_TAG || steps.pr-metadata-comment.outputs.COMPARISON_TAG }}

# below are used in the experiment/analyze jobs

cpus: ${{ steps.system.outputs.CPUS }}

memory: ${{ steps.system.outputs.MEMORY }}

vector-cpus: ${{ steps.system.outputs.VECTOR_CPUS }}

replicas: ${{ steps.experimental-meta.outputs.REPLICAS }}

warmup-seconds: ${{ steps.experimental-meta.outputs.WARMUP_SECONDS }}

total-samples: ${{ steps.experimental-meta.outputs.TOTAL_SAMPLES }}

p-value: ${{ steps.experimental-meta.outputs.P_VALUE }}

smp-version: ${{ steps.experimental-meta.outputs.SMP_CRATE_VERSION }}

lading-version: ${{ steps.experimental-meta.outputs.LADING_VERSION }}

steps:

- uses: actions/checkout@v3

with:

fetch-depth: 1000

# If triggered by issue comment, the event payload doesn't directly contain the head and base sha from the PR.

# But, we can retrieve this info from some commands.

- name: Get PR metadata (issue_comment)

id: pr-metadata-comment

if: github.event_name == 'issue_comment'

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

export PR_NUMBER=${{ github.event.issue.number }}

echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT

gh pr checkout ${{ github.event.issue.number }}

export BASELINE_SHA=$(git merge-base master HEAD)

echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT

export COMPARISON_SHA=$(git rev-parse HEAD)

echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT

export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}"

echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT

export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}"

echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT

echo "pr number is: ${PR_NUMBER}"

echo "baseline sha is: ${BASELINE_SHA}"

echo "baseline tag is: ${BASELINE_TAG}"

echo "comparison sha is: ${COMPARISON_SHA}"

echo "comparison tag is: ${COMPARISON_TAG}"

if [ "${BASELINE_SHA}" = "" ] ; then

echo "BASELINE_SHA not found, exiting."

exit 1

fi

if [ "${COMPARISON_SHA}" = "" ] ; then

echo "COMPARISON_SHA not found, exiting."

exit 1

fi

if [ "${PR_NUMBER}" = "" ] ; then

echo "PR_NUMBER not found, exiting."

exit 1

fi

# If triggered by merge queue, the PR number is not available in the payload. While we restrict the number of PRs in the

# queue to 1, we can get the PR number by parsing the merge queue temp branch's ref.

- name: Get PR metadata (merge queue)

id: pr-metadata-merge-queue

if: github.event_name != 'issue_comment'

run: |

export PR_NUMBER=$(echo "${{ github.ref }}" | sed -n 's|^refs/heads/gh-readonly-queue/master/pr-\([0-9]*\)-.*$|\1|p')

echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT

export BASELINE_SHA=${{ github.event.merge_group.base_sha }}

echo "BASELINE_SHA=${BASELINE_SHA}" >> $GITHUB_OUTPUT

export COMPARISON_SHA=${{ github.event.merge_group.head_sha }}

echo "COMPARISON_SHA=${COMPARISON_SHA}" >> $GITHUB_OUTPUT

export BASELINE_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${BASELINE_SHA}"

echo "BASELINE_TAG=${BASELINE_TAG}" >> $GITHUB_OUTPUT

export COMPARISON_TAG="${PR_NUMBER}-${COMPARISON_SHA}-${COMPARISON_SHA}"

echo "COMPARISON_TAG=${COMPARISON_TAG}" >> $GITHUB_OUTPUT

echo "pr number is: ${PR_NUMBER}"

echo "baseline sha is: ${BASELINE_SHA}"

echo "baseline tag is: ${BASELINE_TAG}"

echo "comparison sha is: ${COMPARISON_SHA}"

echo "comparison tag is: ${COMPARISON_TAG}"

if [ "${BASELINE_SHA}" = "" ] ; then

echo "BASELINE_SHA not found, exiting."

exit 1

fi

if [ "${COMPARISON_SHA}" = "" ] ; then

echo "COMPARISON_SHA not found, exiting."

exit 1

fi

if [ "${PR_NUMBER}" = "" ] ; then

echo "PR_NUMBER not found, exiting."

exit 1

fi

- name: Setup experimental metadata

id: experimental-meta

run: |

export WARMUP_SECONDS="45"

export REPLICAS="10"

export TOTAL_SAMPLES="600"

export P_VALUE="0.1"

export SMP_CRATE_VERSION="0.10.0"

export LADING_VERSION="0.19.1"

echo "warmup seconds: ${WARMUP_SECONDS}"

echo "replicas: ${REPLICAS}"

echo "total samples: ${TOTAL_SAMPLES}"

echo "regression p-value: ${P_VALUE}"

echo "smp crate version: ${SMP_CRATE_VERSION}"

echo "lading version: ${LADING_VERSION}"

echo "WARMUP_SECONDS=${WARMUP_SECONDS}" >> $GITHUB_OUTPUT

echo "REPLICAS=${REPLICAS}" >> $GITHUB_OUTPUT

echo "TOTAL_SAMPLES=${TOTAL_SAMPLES}" >> $GITHUB_OUTPUT

echo "P_VALUE=${P_VALUE}" >> $GITHUB_OUTPUT

echo "SMP_CRATE_VERSION=${SMP_CRATE_VERSION}" >> $GITHUB_OUTPUT

echo "LADING_VERSION=${LADING_VERSION}" >> $GITHUB_OUTPUT

- name: Setup system details

id: system

run: |

export CPUS="7"

export MEMORY="30g"

export VECTOR_CPUS="4"

echo "cpus total: ${CPUS}"

echo "memory total: ${MEMORY}"

echo "vector cpus: ${VECTOR_CPUS}"

echo "CPUS=${CPUS}" >> $GITHUB_OUTPUT

echo "MEMORY=${MEMORY}" >> $GITHUB_OUTPUT

echo "VECTOR_CPUS=${VECTOR_CPUS}" >> $GITHUB_OUTPUT

- name: (PR comment) Get PR branch

if: ${{ github.event_name == 'issue_comment' }}

uses: xt0rted/pull-request-comment-branch@v2

id: comment-branch

- name: (PR comment) Set latest commit status as pending

if: ${{ github.event_name == 'issue_comment' }}

uses: myrotvorets/set-commit-status-action@v2.0.0

with:

sha: ${{ steps.pr-metadata-comment.outputs.COMPARISON_SHA }}

token: ${{ secrets.GITHUB_TOKEN }}

context: Regression Detection Suite

status: pending

##

## BUILD

##

build-baseline:

name: Build baseline Vector container

runs-on: [linux, ubuntu-20.04-4core]

needs:

- compute-metadata

steps:

- uses: colpal/actions-clean@v1

- uses: actions/checkout@v3

- uses: actions/checkout@v3

with:

ref: ${{ needs.compute-metadata.outputs.baseline-sha }}

path: baseline-vector

- name: Set up Docker Buildx

id: buildx

uses: docker/setup-buildx-action@v3.0.0

- name: Build 'vector' target image

uses: docker/build-push-action@v5.0.0

with:

context: baseline-vector/

cache-from: type=gha

cache-to: type=gha,mode=max

file: regression/Dockerfile

builder: ${{ steps.buildx.outputs.name }}

outputs: type=docker,dest=${{ runner.temp }}/baseline-image.tar

tags: |

vector:${{ needs.compute-metadata.outputs.baseline-tag }}

- name: Upload image as artifact

uses: actions/upload-artifact@v3

with:

name: baseline-image

path: "${{ runner.temp }}/baseline-image.tar"

build-comparison:

name: Build comparison Vector container

runs-on: [linux, ubuntu-20.04-4core]

needs:

- compute-metadata

steps:

- uses: colpal/actions-clean@v1

- uses: actions/checkout@v3

- uses: actions/checkout@v3

with:

ref: ${{ needs.compute-metadata.outputs.comparison-sha }}

path: comparison-vector

- name: Set up Docker Buildx

id: buildx

uses: docker/setup-buildx-action@v3.0.0

- name: Build 'vector' target image

uses: docker/build-push-action@v5.0.0

with:

context: comparison-vector/

cache-from: type=gha

cache-to: type=gha,mode=max

file: regression/Dockerfile

builder: ${{ steps.buildx.outputs.name }}

outputs: type=docker,dest=${{ runner.temp }}/comparison-image.tar

tags: |

vector:${{ needs.compute-metadata.outputs.comparison-tag }}

- name: Upload image as artifact

uses: actions/upload-artifact@v3

with:

name: comparison-image

path: "${{ runner.temp }}/comparison-image.tar"

confirm-valid-credentials:

name: Confirm AWS credentials are minimally valid

runs-on: ubuntu-22.04

needs:

- compute-metadata

steps:

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Download SMP binary

run: |

aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp

##

## SUBMIT

##

upload-baseline-image-to-ecr:

name: Upload baseline images to ECR

runs-on: ubuntu-22.04

needs:

- compute-metadata

- confirm-valid-credentials

- build-baseline

steps:

- name: 'Download baseline image'

uses: actions/download-artifact@v3

with:

name: baseline-image

- name: Load baseline image

run: |

docker load --input baseline-image.tar

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Login to Amazon ECR

id: login-ecr

uses: aws-actions/amazon-ecr-login@v2

- name: Docker Login to ECR

uses: docker/login-action@v3

with:

registry: ${{ steps.login-ecr.outputs.registry }}

- name: Tag & push baseline image

run: |

docker tag vector:${{ needs.compute-metadata.outputs.baseline-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }}

docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }}

upload-comparison-image-to-ecr:

name: Upload comparison images to ECR

runs-on: ubuntu-22.04

needs:

- compute-metadata

- confirm-valid-credentials

- build-comparison

steps:

- name: 'Download comparison image'

uses: actions/download-artifact@v3

with:

name: comparison-image

- name: Load comparison image

run: |

docker load --input comparison-image.tar

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Login to Amazon ECR

id: login-ecr

uses: aws-actions/amazon-ecr-login@v2

- name: Docker Login to ECR

uses: docker/login-action@v3

with:

registry: ${{ steps.login-ecr.outputs.registry }}

- name: Tag & push comparison image

run: |

docker tag vector:${{ needs.compute-metadata.outputs.comparison-tag }} ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }}

docker push ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }}

submit-job:

name: Submit regression job

runs-on: ubuntu-22.04

needs:

- compute-metadata

- upload-baseline-image-to-ecr

- upload-comparison-image-to-ecr

steps:

- name: Check status, in-progress

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='pending' \

-f description='Experiments submitted to the Regression Detection cluster.' \

-f context='Regression Detection Suite / submission' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- uses: actions/checkout@v3

with:

ref: ${{ needs.compute-metadata.outputs.comparison-sha }}

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Login to Amazon ECR

id: login-ecr

uses: aws-actions/amazon-ecr-login@v2

- name: Download SMP binary

run: |

aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp

- name: Submit job

env:

RUST_LOG: info

run: |

chmod +x ${{ runner.temp }}/bin/smp

${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \

--lading-version ${{ needs.compute-metadata.outputs.lading-version }} \

--total-samples ${{ needs.compute-metadata.outputs.total-samples }} \

--warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \

--replicas ${{ needs.compute-metadata.outputs.replicas }} \

--baseline-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.baseline-tag }} \

--comparison-image ${{ steps.login-ecr.outputs.registry }}/${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }}-vector:${{ needs.compute-metadata.outputs.comparison-tag }} \

--baseline-sha ${{ needs.compute-metadata.outputs.baseline-sha }} \

--comparison-sha ${{ needs.compute-metadata.outputs.comparison-sha }} \

--target-command "/usr/local/bin/vector" \

--target-config-dir ${{ github.workspace }}/regression/ \

--target-cpu-allotment "${{ needs.compute-metadata.outputs.cpus }}" \

--target-memory-allotment "${{ needs.compute-metadata.outputs.memory }}" \

--target-environment-variables "VECTOR_THREADS=${{ needs.compute-metadata.outputs.vector-cpus }},VECTOR_REQUIRE_HEALTHY=true" \

--target-name vector \

--submission-metadata ${{ runner.temp }}/submission-metadata

- uses: actions/upload-artifact@v3

with:

name: vector-submission-metadata

path: ${{ runner.temp }}/submission-metadata

- name: Await job

timeout-minutes: 120

env:

RUST_LOG: info

run: |

chmod +x ${{ runner.temp }}/bin/smp

${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \

--wait \

--wait-delay-seconds 60 \

--wait-timeout-minutes 90 \

--submission-metadata ${{ runner.temp }}/submission-metadata

- name: Handle cancellation if necessary

if: ${{ cancelled() }}

env:

RUST_LOG: info

run: |

chmod +x ${{ runner.temp }}/bin/smp

${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \

--submission-metadata ${{ runner.temp }}/submission-metadata

- name: Check status, cancelled

if: ${{ cancelled() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='failure' \

-f description='Experiments submitted to the Regression Detection cluster cancelled.' \

-f context='Regression Detection Suite / submission' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, success

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='success' \

-f description='Experiments submitted to the Regression Detection cluster successfully.' \

-f context='Regression Detection Suite / submission' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, failure

if: ${{ failure() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='success' \

-f description='Experiments submitted to the Regression Detection Suite failed.' \

-f context='Regression Detection Suite / submission' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

##

## ANALYZE

##

detect-regression:

name: Determine regression status

runs-on: ubuntu-22.04

needs:

- submit-job

- compute-metadata

steps:

- uses: actions/checkout@v3

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Download SMP binary

run: |

aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp

- name: Download submission metadata

uses: actions/download-artifact@v3

with:

name: vector-submission-metadata

path: ${{ runner.temp }}/

- name: Determine if PR introduced a regression

env:

RUST_LOG: info

run: |

chmod +x ${{ runner.temp }}/bin/smp

${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \

--submission-metadata ${{ runner.temp }}/submission-metadata

- name: Check status, cancelled

if: ${{ cancelled() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='failure' \

-f description='Analyze experimental results from Regression Detection Suite cancelled.' \

-f context='Regression Detection Suite / detect-regression' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, success

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='success' \

-f description='Analyze experimental results from Regression Detection Suite succeeded.' \

-f context='Regression Detection Suite / detect-regression' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, failure

if: ${{ failure() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='failure' \

-f description='Analyze experimental results from Regression Detection Suite failed.' \

-f context='Regression Detection Suite / detect-regression' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

analyze-experiment:

name: Download regression analysis & upload report

runs-on: ubuntu-22.04

needs:

- submit-job

- compute-metadata

steps:

- name: Check status, in-progress

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='pending' \

-f description='Analyze experimental results from Regression Detection Suite.' \

-f context='Regression Detection Suite / analyze-experiment' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- uses: actions/checkout@v3

with:

ref: ${{ needs.compute-metadata.outputs.comparison-sha }}

- name: Configure AWS Credentials

uses: aws-actions/configure-aws-credentials@v4.0.1

with:

aws-access-key-id: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_ACCESS_KEY_ID }}

aws-secret-access-key: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_BOT_SECRET_ACCESS_KEY }}

aws-region: us-west-2

- name: Download SMP binary

run: |

aws s3 cp s3://smp-cli-releases/v${{ needs.compute-metadata.outputs.smp-version }}/x86_64-unknown-linux-gnu/smp ${{ runner.temp }}/bin/smp

- name: Download submission metadata

uses: actions/download-artifact@v3

with:

name: vector-submission-metadata

path: ${{ runner.temp }}/

- name: Sync regression report to local system

env:

RUST_LOG: info

run: |

chmod +x ${{ runner.temp }}/bin/smp

${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \

--submission-metadata ${{ runner.temp }}/submission-metadata \

--output-path "${{ runner.temp }}/outputs"

- name: Read regression report

id: read-analysis

uses: juliangruber/read-file-action@v1

with:

path: ${{ runner.temp }}/outputs/report.html

- name: Post report to PR

uses: peter-evans/create-or-update-comment@v3

with:

issue-number: ${{ needs.compute-metadata.outputs.pr-number }}

edit-mode: append

body: ${{ steps.read-analysis.outputs.content }}

- name: Upload regression report to artifacts

uses: actions/upload-artifact@v3

with:

name: capture-artifacts

path: ${{ runner.temp }}/outputs/*

- name: Check status, cancelled

if: ${{ cancelled() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='failure' \

-f description='Analyze experimental results from Regression Detection Suite cancelled.' \

-f context='Regression Detection Suite / analyze-experiment' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, success

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='success' \

-f description='Analyze experimental results from Regression Detection Suite succeeded.' \

-f context='Regression Detection Suite / analyze-experiment' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

- name: Check status, failure

if: ${{ failure() }}

env:

GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

run: |

gh api \

--method POST \

-H "Accept: application/vnd.github+json" \

/repos/${{ github.repository }}/statuses/${{ needs.compute-metadata.outputs.comparison-sha }} \

-f state='failure' \

-f description='Analyze experimental results from Regression Detection Suite failed.' \

-f context='Regression Detection Suite / analyze-experiment' \

-f target_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

# This job always runs- if an issue_comment triggered it, we need to update the check status in the PR,

# and if a pull_request triggered it, we need to flag the check status as a success.

regression-detection-suite:

name: Regression Detection Suite

runs-on: ubuntu-latest

if: always()

needs:

- compute-metadata

- build-baseline

- build-comparison

- confirm-valid-credentials

- upload-baseline-image-to-ecr

- upload-comparison-image-to-ecr

- submit-job

- detect-regression

- analyze-experiment

env:

FAILED: ${{ contains(needs.*.result, 'failure') }}

steps:

- name: (PR comment) Get PR branch

if: github.event_name == 'issue_comment'

uses: xt0rted/pull-request-comment-branch@v2

id: comment-branch

- name: (PR comment) Submit PR result as failed

if: github.event_name == 'issue_comment' && env.FAILED == 'true'

uses: myrotvorets/set-commit-status-action@v2.0.0

with:

sha: ${{ steps.comment-branch.outputs.head_sha }}

token: ${{ secrets.GITHUB_TOKEN }}

context: Regression Detection Suite

status: 'failure'

- name: (PR comment) Submit PR result as success

if: github.event_name == 'issue_comment' && env.FAILED != 'true'

uses: myrotvorets/set-commit-status-action@v2.0.0

with:

sha: ${{ steps.comment-branch.outputs.head_sha }}

token: ${{ secrets.GITHUB_TOKEN }}

context: Regression Detection Suite

status: 'success'

- name: (PR) Indicate why skipped

if: github.event_name == 'pull_request'

run: |

echo "### Workflow skipped" >> $GITHUB_STEP_SUMMARY

echo "This workflow doesn't run on PR's automatically." >> $GITHUB_STEP_SUMMARY

echo "To trigger a run, leave a comment with \`/ci-run-regression\`" >> $GITHUB_STEP_SUMMARY

- name: exit

run: |

echo "failed=${{ env.FAILED }}"

if [[ "$FAILED" == "true" ]] ; then

exit 1

else

exit 0

fi