feast-dev · woop · Jul 26, 2020 · Jul 25, 2020 · Jul 25, 2020 · Jul 25, 2020
diff --git a/.github/workflows/load_test.yml b/.github/workflows/load_test.yml
@@ -0,0 +1,16 @@
+name: load test
+
+on: [push, pull_request]
+
+jobs:
+  load-test:
+    runs-on: [self-hosted, load]
+    name: load-test
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run load test
+        run:  make test-load
+      - uses: actions/upload-artifact@v2
+        with:
+          name: load-test-results
+          path: load-test-output/
diff --git a/Makefile b/Makefile
@@ -177,5 +177,11 @@ build-html: clean-html
 	cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python
 
 # Versions
+
 lint-versions:
 	./infra/scripts/validate-version-consistency.sh
+
+# Performance
+
+test-load:
+	./infra/scripts/test-load.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+set -e
+
+echo "
+============================================================
+Running Load Tests
+============================================================
+"
+
+clean_up() {
+  ARG=$?
+
+  # Shut down docker-compose images
+  cd "${PROJECT_ROOT_DIR}"/infra/docker-compose
+
+  docker-compose \
+    -f docker-compose.yml \
+    -f docker-compose.online.yml down
+
+  # Remove configuration file
+  rm .env
+
+  exit $ARG
+}
+
+CURRENT_SHA=$(git rev-parse HEAD)
+
+if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
+    export CURRENT_SHA=$(cat "$GITHUB_EVENT_PATH" | jq -r .pull_request.head.sha)
+fi
+
+export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
+export COMPOSE_INTERACTIVE_NO_CLI=1
+
+# Wait for docker images to be available
+"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-docker-images.sh "${CURRENT_SHA}"
+
+# Clean up Docker Compose if failure
+trap clean_up EXIT
+
+# Create Docker Compose configuration file
+cd "${PROJECT_ROOT_DIR}"/infra/docker-compose/
+cp .env.sample .env
+
+# Start Docker Compose containers
+FEAST_VERSION=${CURRENT_SHA} docker-compose -f docker-compose.yml -f docker-compose.online.yml up -d
+
+# Get Jupyter container IP address
+export JUPYTER_DOCKER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jupyter_1)
+
+# Print Jupyter container information
+docker inspect feast_jupyter_1
+docker logs feast_jupyter_1
+
+# Wait for Jupyter Notebook Container to come online
+"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${JUPYTER_DOCKER_CONTAINER_IP_ADDRESS}:8888 --timeout=60
+
+# Get Feast Core container IP address
+export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_core_1)
+
+# Wait for Feast Core to be ready
+"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120
+
+# Get Feast Online Serving container IP address
+export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online-serving_1)
+
+# Wait for Feast Online Serving to be ready
+"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120
+
+# Ingest data into Feast
+pip install --user matplotlib feast pytz matplotlib --upgrade
+python "${PROJECT_ROOT_DIR}"/tests/load/ingest.py "${FEAST_CORE_CONTAINER_IP_ADDRESS}":6565  "${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}":6566
+
+# Download load test tool and proxy
+cd $(mktemp -d)
+wget -c https://github.com/feast-dev/feast-load-test-proxy/releases/download/v0.1.1/feast-load-test-proxy_0.1.1_Linux_x86_64.tar.gz -O - | tar -xz
+git clone https://github.com/giltene/wrk2.git
+cd wrk2
+make
+cd ..
+cp wrk2/wrk .
+
+# Start load test server
+LOAD_FEAST_SERVING_HOST=${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS} LOAD_FEAST_SERVING_PORT=6566 ./feast-load-test-proxy &
+sleep 5
+
+# Run load tests
+./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/echo
+./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_20rps
+./wrk -t2 -c10 -d30s -R50 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_50rps
+./wrk -t2 -c10 -d30s -R250 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_250rps
+./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_20rps
+./wrk -t2 -c10 -d30s -R50 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_50rps
+./wrk -t2 -c10 -d30s -R250 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_250rps
+
+# Print load test results
+cat $(ls -lah | grep load_test_results | awk '{print $9}' | tr '\n' ' ')
+
+# Create hdr-plot of load tests
+export PLOT_FILE_NAME="load_test_graph_${CURRENT_SHA}"_$(date "+%Y%m%d-%H%M%S").png
+python $PROJECT_ROOT_DIR/tests/load/hdr_plot.py --output "$PLOT_FILE_NAME" --title "Load test: ${CURRENT_SHA}" $(ls -lah | grep load_test_results | awk '{print $9}' | tr '\n' ' ')
+
+# Persist artifact
+mkdir -p "${PROJECT_ROOT_DIR}"/load-test-output/
+cp "${PLOT_FILE_NAME}" "${PROJECT_ROOT_DIR}"/load-test-output/
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# This script will block until both the Feast Serving and Feast Core docker images are available for use for a specific tag.
+#
+
+[[ -z "$1" ]] && { echo "Please pass the Git SHA as the first parameter" ; exit 1; }
+
+GIT_SHA=$1
+
+# Set allowed failure count
+poll_count=0
+maximum_poll_count=150
+
+# Wait for Feast Core to be available on GCR
+until docker pull gcr.io/kf-feast/feast-core:"${GIT_SHA}"
+do
+  # Exit when we have tried enough times
+  if [[ "$poll_count" -gt "$maximum_poll_count" ]]; then
+       exit 1
+  fi
+
+  # Sleep and increment counter on failure
+  echo "gcr.io/kf-feast/feast-core:${GIT_SHA} could not be found";
+  sleep 5;
+  ((poll_count++))
+done
+
+# Wait for Feast Serving to be available on GCR
+until docker pull gcr.io/kf-feast/feast-serving:"${GIT_SHA}"
+do
+  # Exit when we have tried enough times
+  if [[ "$poll_count" -gt "$maximum_poll_count" ]]; then
+       exit 1
+  fi
+
+  # Sleep and increment counter on failure
+  echo "gcr.io/kf-feast/feast-serving:${GIT_SHA} could not be found";
+  sleep 5;
+  ((poll_count++))
+done
diff --git a/tests/load/hdr_plot.py b/tests/load/hdr_plot.py
@@ -0,0 +1,118 @@
+#
+# hdr-plot.py v0.2.0 - A simple HdrHistogram plotting script.
+# Copyright © 2018 Bruno Bonacci - Distributed under the Apache License v 2.0
+#
+# usage: hdr-plot.py [-h] [--output OUTPUT] [--title TITLE] [--nobox] files [files ...]
+#
+# A standalone plotting script for https://github.com/giltene/wrk2 and
+#  https://github.com/HdrHistogram/HdrHistogram.
+#
+# This is just a quick and unsophisticated script to quickly plot the
+# HdrHistograms directly from the output of `wkr2` benchmarks.
+
+import argparse
+import re
+import pandas as pd
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+
+regex = re.compile(r'\s+([0-9.]+)\s+([0-9.]+)\s+([0-9.]+)\s+([0-9.]+)')
+filename = re.compile(r'(.*/)?([^.]*)(\.\w+\d+)?')
+
+
+def parse_percentiles(file):
+    lines = [line for line in open(file) if re.match(regex, line)]
+    values = [re.findall(regex, line)[0] for line in lines]
+    pctles = [(float(v[0]), float(v[1]), int(v[2]), float(v[3])) for v in values]
+    percentiles = pd.DataFrame(pctles, columns=['Latency', 'Percentile', 'TotalCount', 'inv-pct'])
+    return percentiles
+
+
+def parse_files(files):
+    return [parse_percentiles(file) for file in files]
+
+
+def info_text(name, data):
+    textstr = '%-18s\n------------------\n%-6s = %6.2f ms\n%-6s = %6.2f ms\n%-6s = %6.2f ms\n' % (
+        name,
+        "min", data['Latency'].min(),
+        "median", data[data["Percentile"] == 0.5]["Latency"],
+        "max", data['Latency'].max())
+    return textstr
+
+
+def info_box(ax, text):
+    props = dict(boxstyle='round', facecolor='lightcyan', alpha=0.5)
+
+    # place a text box in upper left in axes coords
+    ax.text(0.05, 0.95, text, transform=ax.transAxes,
+            verticalalignment='top', bbox=props, fontname='monospace')
+
+
+def plot_summarybox(ax, percentiles, labels):
+    # add info box to the side
+    textstr = '\n'.join([info_text(labels[i], percentiles[i]) for i in range(len(labels))])
+    info_box(ax, textstr)
+
+
+def plot_percentiles(percentiles, labels):
+    y_range = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
+    x_range = [0.25, 0.5, 0.9, 0.99, 0.999, 0.9999, 0.99999, 0.999999]
+
+    fig, ax = plt.subplots(figsize=(24, 16))
+    plt.ylim(0, 50)
+    # plot values
+    for data in percentiles:
+        ax.plot(data['Percentile'], data['Latency'])
+
+    # set axis and legend
+    ax.grid()
+    ax.set(xlabel='Percentile',
+           ylabel='Latency (milliseconds)',
+           title='Latency Percentiles (lower is better)')
+    ax.set_xscale('logit')
+    plt.yticks(y_range)
+    plt.xticks(x_range)
+    majors = ["25%", "50%", "90%", "99%", "99.9%", "99.99%", "99.999%", "99.9999%"]
+    ax.xaxis.set_major_formatter(ticker.FixedFormatter(majors))
+    ax.xaxis.set_minor_formatter(ticker.NullFormatter())
+    plt.legend(bbox_to_anchor=(0., 1.02, 1., .102),
+               loc=3, ncol=2, borderaxespad=0.,
+               labels=labels)
+
+    return fig, ax
+
+
+def arg_parse():
+    parser = argparse.ArgumentParser(description='Plot HDRHistogram latencies.')
+    parser.add_argument('files', nargs='+', help='list HDR files to plot')
+    parser.add_argument('--output', default='latency.png',
+                        help='Output file name (default: latency.png)')
+    parser.add_argument('--title', default='', help='The plot title.')
+    parser.add_argument("--nobox", help="Do not plot summary box",
+                        action="store_true")
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    # print command line arguments
+    args = arg_parse()
+
+    # load the data and create the plot
+    pct_data = parse_files(args.files)
+    labels = [re.findall(filename, file)[0][1] for file in args.files]
+    # plotting data
+    fig, ax = plot_percentiles(pct_data, labels)
+    # plotting summary box
+    if not args.nobox:
+        plot_summarybox(ax, pct_data, labels)
+    # add title
+    plt.suptitle(args.title)
+    # save image
+    plt.savefig(args.output)
+    print("Wrote: " + args.output)
+
+
+if __name__ == "__main__":
+    main()