Skip to content

Commit 9d977be

Browse files
odeke-emodeke-em
authored
.kokoro: parallelize tests across single build (#405)
* parallelize tests on multiple cores across single build * install Django test requirements Uses Go code and workers to run 2 tests at a time, scheduled across the various cores. When a worker finishes running, its goroutine dies and another one gets spawned to handle the work availability. Updates #289
1 parent 7ffc014 commit 9d977be

File tree

4 files changed

+109
-16
lines changed

4 files changed

+109
-16
lines changed

‎packages/django-google-spanner/.kokoro/build.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,5 +39,14 @@ python3.6 -m flake8
3939

4040
# Run with the Django test apps.
4141
export RUNNING_SPANNER_BACKEND_TESTS=1
42-
export DJANGO_TEST_APPS="backends basic bulk_create choices custom_columns"
43-
bash django_test_suite.sh
42+
export DJANGO_TEST_APPS="admin_changelist admin_custom_urls admin_docs admin_inlines admin_ordering aggregation aggregation_regress annotations backends basic bulk_create cache choices custom_columns indexes inline_formsets introspection invalid_models_tests known_related_objects lookup max_lengths m2m_and_m2o m2m_intermediary m2m_multiple m2m_recursive m2m_regress m2m_signals m2m_through m2m_through_regress m2o_recursive managers_regress many_to_many many_to_one many_to_one_null max_lengths migrate_signals migrations.test_operations migration_test_data_persistence"
43+
44+
pip3 install .
45+
mkdir -p django_tests && git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git django_tests/django
46+
# cd django_tests/django && pip3 install -e .; cd ../../
47+
48+
# Install dependencies for Django tests.
49+
sudo apt-get update
50+
apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
51+
cd django_tests/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
52+
./bin/parallelize_tests_linux

‎packages/django-google-spanner/bin/parallelize_tests_linux‎

6.78 MB
Binary file not shown.

‎packages/django-google-spanner/django_test_suite.sh‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,6 @@ INSTANCE=${SPANNER_TEST_INSTANCE:-django-tests}
1818
PROJECT=${PROJECT_ID:-appdev-soda-spanner-staging}
1919
SETTINGS_FILE="$TEST_DBNAME-settings"
2020

21-
checkout_django() {
22-
mkdir -p django_tests && cd django_tests
23-
git clone --depth 1 --single-branch --branch spanner-2.2.x https://github.com/timgraham/django.git
24-
cd django && pip3 install -e .
25-
# pip3 install -r tests/requirements/py3.txt
26-
}
27-
2821
create_settings() {
2922
cat << ! > "$SETTINGS_FILE.py"
3023
DATABASES = {
@@ -49,16 +42,10 @@ PASSWORD_HASHERS = [
4942
}
5043

5144
run_django_tests() {
52-
cd tests
45+
cd django_tests/django/tests
5346
create_settings
5447
echo -e "\033[32mRunning Django tests $TEST_APPS\033[00m"
5548
python3 runtests.py $TEST_APPS --verbosity=2 --noinput --settings $SETTINGS_FILE
5649
}
5750

58-
install_django_spanner() {
59-
pip3 install .
60-
}
61-
62-
install_django_spanner
63-
checkout_django
6451
run_django_tests

‎packages/django-google-spanner/parallelize_tests.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
// Copyright 2020 Google LLC.
2+
//
3+
// Use of this source code is governed by a BSD-style
4+
// license that can be found in the LICENSE file or at
5+
// https://developers.google.com/open-source/licenses/bsd
6+
7+
package main
8+
9+
import (
10+
"context"
11+
"errors"
12+
"math/rand"
13+
"os"
14+
"os/exec"
15+
"os/signal"
16+
"runtime"
17+
"strings"
18+
"sync"
19+
20+
"github.com/orijtech/otils"
21+
)
22+
23+
func main() {
24+
testApps := otils.NonEmptyStrings(strings.Split(os.Getenv("DJANGO_TEST_APPS"), " ")...)
25+
if len(testApps) == 0 {
26+
panic("No DJANGO_TEST_APPS passed in")
27+
}
28+
29+
rand.Shuffle(len(testApps), func(i, j int) {
30+
testApps[i], testApps[j] = testApps[j], testApps[i]
31+
})
32+
33+
// Otherwise, we'll parallelize the builds.
34+
nProcs := runtime.GOMAXPROCS(0)
35+
println("nProcs", nProcs)
36+
37+
shutdownCtx, cancel := context.WithCancel(context.Background())
38+
defer cancel()
39+
40+
// Gracefully shutdown on Ctrl+C.
41+
sigCh := make(chan os.Signal, 1)
42+
signal.Notify(sigCh, os.Interrupt)
43+
go func() {
44+
<-sigCh
45+
cancel()
46+
}()
47+
48+
var wg sync.WaitGroup
49+
defer wg.Wait()
50+
51+
// The number of Django apps to run per goroutine.
52+
nAppsPerG := 4
53+
if len(testApps) <= nAppsPerG {
54+
nAppsPerG = 1
55+
} else {
56+
nAppsPerG = len(testApps) / (nAppsPerG * nProcs)
57+
}
58+
println("apps per G: ", nAppsPerG)
59+
60+
if nAppsPerG == 0 {
61+
nAppsPerG = 2
62+
}
63+
64+
sema := make(chan bool, nProcs)
65+
// Now run the tests in parallel.
66+
for i := 0; i < len(testApps); i += nAppsPerG {
67+
apps := testApps[i : i+nAppsPerG]
68+
wg.Add(1)
69+
sema <- true
70+
71+
go func(wg *sync.WaitGroup, apps []string) {
72+
defer func() {
73+
<-sema
74+
wg.Done()
75+
}()
76+
77+
if len(apps) == 0 {
78+
return
79+
}
80+
if err := runTests(shutdownCtx, apps, "django_test_suite.sh"); err != nil {
81+
panic(err)
82+
}
83+
}(&wg, apps)
84+
}
85+
}
86+
87+
func runTests(ctx context.Context, djangoApps []string, testSuiteScriptPath string) error {
88+
if len(djangoApps) == 0 {
89+
return errors.New("Expected at least one app")
90+
}
91+
92+
cmd := exec.CommandContext(ctx, "bash", testSuiteScriptPath)
93+
cmd.Env = append(os.Environ(), `DJANGO_TEST_APPS=`+strings.Join(djangoApps, " ")+``)
94+
cmd.Stderr = os.Stderr
95+
cmd.Stdout = os.Stdout
96+
return cmd.Run()
97+
}

0 commit comments

Comments
 (0)