Skip to content

Commit f6da52c

Browse files
committed
Merge branch 'yanhuil/benchmark_dashboard' of https://github.com/GoogleCloudPlatform/python-runtime into yanhuil/benchmark_dashboard
2 parents 4a2e096 + cb0fad1 commit f6da52c

3 files changed

Lines changed: 21 additions & 18 deletions

File tree

‎tests/benchmark/Dockerfile.in‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@ RUN mkdir /result
1717
RUN pyperformance run --debug-single-value --python=python2.7 -o /result/py2.7.json
1818
RUN pyperformance run --debug-single-value --python=python3.4 -o /result/py3.4.json
1919
RUN pyperformance run --debug-single-value --python=python3.5 -o /result/py3.5.json
20-
RUN pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.json
20+
RUN if [ -e "/opt/python3.6/bin/python3.6" ]; then pyperformance run --debug-single-value --python=python3.6 -o /result/py3.6.json; fi
2121

2222
RUN pyperformance compare /result/py2.7.json /result/py3.4.json --output_style table
2323
RUN pyperformance compare /result/py3.4.json /result/py3.5.json --output_style table
24-
RUN pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table
25-
24+
RUN if [ -e "/result/py3.6.json" ]; then pyperformance compare /result/py3.5.json /result/py3.6.json --output_style table; fi

‎tests/benchmark/benchmark_between_releases.sh‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ rm Dockerfile
1717
echo "Successfully built images"
1818

1919
# Create folders to hold the files
20-
mkdir $TAG1
21-
mkdir $TAG2
20+
mkdir "$TAG1"
21+
mkdir "$TAG2"
2222

2323
# Start running the containers and copy the benchmark result for python versions from container to host
2424
docker run -it --name benchmark_1 -h CONTAINER1 -v "${PWD}"/"$TAG1":/export benchmark_1 /bin/bash -c "cp /result/py*.json /export/"
@@ -30,7 +30,11 @@ echo "Start benchmarking the python interpreter performance between the two rele
3030
pyperformance compare "$TAG1"/py2.7.json "$TAG2"/py2.7.json --output_style table > py2.7_res
3131
pyperformance compare "$TAG1"/py3.4.json "$TAG2"/py3.4.json --output_style table > py3.4_res
3232
pyperformance compare "$TAG1"/py3.5.json "$TAG2"/py3.5.json --output_style table > py3.5_res
33-
pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res
33+
34+
# Check if the python3.6 benchmark result exists
35+
if [[ ( -e '"$TAG1"/py3.6.json' ) && ( -e '"$TAG2"/py3.6.json' ) ]]; then
36+
pyperformance compare "$TAG1"/py3.6.json "$TAG2"/py3.6.json --output_style table > py3.6_res;
37+
fi
3438

3539
echo "Start extracting data and generating CSV file, then upload to Cloud Storage and insert to Big Query table"
3640

@@ -43,22 +47,25 @@ for path_to_file in $TAG2/*.json; do
4347
python generate_csv.py --filename $path_to_file --tag $TAG2
4448
done
4549

46-
# Get the list of existed release data on Cloud Storage and skip if the current TAG1 or TAG2 existed in the list
47-
gsutil ls gs://python_runtime_benchmark > existed_releases
50+
# Set the project that hold the cloud storage bucket and big query tables
51+
gcloud config set project cloud-python-runtime-qa
52+
53+
# Get the list of existing release data on Cloud Storage and skip if the current TAG1 or TAG2 existing in the list
54+
gsutil ls gs://python-runtime-benchmark > existing_releases
4855

4956
for container_tag in $TAG1 $TAG2; do
50-
if grep --fixed-strings --quiet "$container_tag" existed_releases; then
51-
echo "Performace data of $container_tag existed, so skip processing it."
57+
if grep --fixed-strings --quiet "$container_tag" existing_releases; then
58+
echo "Performance data of $container_tag existed, so skip processing it."
5259
else
5360
# Upload the CSV files to Cloud Storage
54-
gsutil cp -r $container_tag gs://python_runtime_benchmark
61+
gsutil cp -r $container_tag gs://python-runtime-benchmark
5562
# Load the CSV files from Cloud Storage to Big Query table
5663
# Load the performance data of each function
5764
for path_to_file in $container_tag/py2.7.csv $container_tag/py3.4.csv $container_tag/py3.5.csv; do
58-
bq load benchmark_test.benchmark_functions gs://python_runtime_benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float
65+
bq load benchmark.benchmark_functions gs://python-runtime-benchmark/"$path_to_file" container_tag:string,runtime_version:string,function_name:string,time_used:float,mem_usage:float
5966
done
6067
# Load the average performance data of each runtime version in a release
61-
bq load benchmark_test.benchmark_statistics gs://python_runtime_benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float
68+
bq load benchmark.benchmark_statistics gs://python-runtime-benchmark/"$container_tag"/averages.csv container_tag:string,runtime_version:string,ave_time_used:float,ave_mem_usage:float
6269
fi
6370
done
6471

‎tests/benchmark/generate_csv.py‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,15 @@ def generate_csv(filename, tag):
4040
# Get the function name
4141
func_name = benchmark["metadata"]["name"]
4242
# Get the time used for this function, convert to millisecond
43-
time_used = benchmark["runs"][0]["values"][0] * 1000
43+
time_used = float(benchmark["runs"][0]["values"][0]) * 1000
4444
# Get the memory usage, convert to MB
45-
mem_usage = benchmark["metadata"]["mem_max_rss"] / float(1<<20)
45+
mem_usage = float(benchmark["metadata"]["mem_max_rss"]) / float(1<<20)
4646
line = [tag, runtime_version, func_name, time_used, mem_usage]
4747
# Write to CSV file
4848
csv_writer.writerow(line)
4949
except KeyError:
5050
# Skip the benchmark result if it does not contain the fields we want
5151
pass
52-
output.close()
5352

5453

5554
def get_averages(filename, tag):
@@ -67,7 +66,6 @@ def get_averages(filename, tag):
6766
col_sums = map(sum, zip(*rows_of_data))
6867
# Calculate the average of the two columns by using the sum divided by the total number of lines
6968
averages = [col_sum / len(lines) for col_sum in col_sums]
70-
input.close()
7169

7270
# Get the runtime version from filename
7371
runtime_version = os.path.basename(filename).split(".json")[0]
@@ -79,7 +77,6 @@ def get_averages(filename, tag):
7977
csv_writer.writerow([tag, runtime_version] + averages)
8078
except IOError:
8179
print "Could not write averages to file."
82-
output.close()
8380

8481

8582
def parse_args(argv):

0 commit comments

Comments
 (0)