Skip to content

Commit fb200c6

Browse files
GAumalaedoakes
GAumala
authored and
edoakes
committed
Python integration tests (open-lambda#41)
* Write integration tests in python Write the integration tests found in the Makefile in Python using the unittest framework. * Remove integration tests from makefile * Use check_output instead of call in integration tests When running child processes in integration tests, prefer subprocess.check_output() over subprocess.call(). fix spacing in error messages. * Don't run integration tests in travis CI
1 parent e8c9855 commit fb200c6

5 files changed

Lines changed: 180 additions & 80 deletions

File tree

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,4 +15,4 @@ install:
1515
- sudo ./quickstart/deps.sh
1616

1717
script:
18-
- sudo make test-all
18+
- sudo make

‎Makefile‎

Lines changed: 6 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -8,52 +8,9 @@ POOL_FILES = $(shell find cache-entry)
88

99
TEST_CLUSTER=testing/test-cluster
1010
KILL_WORKER=./bin/admin kill -cluster=$(TEST_CLUSTER);rm -rf $(TEST_CLUSTER)/workers/*
11-
RUN_LAMBDA=curl -XPOST localhost:8080/runLambda
12-
13-
STARTUP_PKGS='{"startup_pkgs": ["parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson"]}'
14-
REGISTRY_DIR='{"registry_dir": "$(abspath testing/registry)"}'
15-
16-
SOCK_NOCACHE='{"sandbox": "sock", "handler_cache_size": 0, "import_cache_size": 0, "cg_pool_size": 10}'
17-
SOCK_HANDLER='{"sandbox": "sock", "handler_cache_size": 10000000, "import_cache_size": 0, "cg_pool_size": 10}'
18-
SOCK_IMPORT='{"sandbox": "sock", "handler_cache_size": 0, "import_cache_size": 10000000, "cg_pool_size": 10}'
19-
SOCK_BOTH='{"sandbox": "sock", "handler_cache_size": 10000000, "import_cache_size": 10000000, "cg_pool_size": 10}'
20-
21-
DOCKER_NOCACHE='{"sandbox": "docker", "handler_cache_size": 0, "import_cache_size": 0, "cg_pool_size": 0}'
22-
DOCKER_HANDLER='{"sandbox": "docker", "handler_cache_size": 10000000, "import_cache_size": 0, "cg_pool_size": 0}'
23-
DOCKER_IMPORT='{"sandbox": "docker", "handler_cache_size": 0, "import_cache_size": 10000000, "cg_pool_size": 0}'
24-
DOCKER_BOTH='{"sandbox": "docker", "handler_cache_size": 10000000, "import_cache_size": 10000000, "cg_pool_size": 0}'
25-
26-
WORKER_TIMEOUT=60
27-
28-
define RUN_TEST=
29-
@echo "Killing worker if running..."
30-
-$(KILL_WORKER)
31-
@echo
32-
@echo "Starting worker..."
33-
./bin/admin setconf -cluster=$(TEST_CLUSTER) CONDITION
34-
./bin/admin workers -cluster=$(TEST_CLUSTER)
35-
@echo
36-
@echo "Waiting for worker to initialize..."
37-
@for i in $$(seq 1 $(WORKER_TIMEOUT)); \
38-
do \
39-
[ $$i -gt 1 ] && sleep 2; \
40-
./bin/admin status -cluster=$(TEST_CLUSTER) 1>/dev/null && s=0 && break || s=$$?; \
41-
done; ([ $$s -eq 0 ] || (echo "Worker failed to initialize after $(WORKER_TIMEOUT)s" && exit 1))
42-
@echo "Worker ready. Requesting lambdas..."
43-
$(RUN_LAMBDA)/echo -d '{}'
44-
@echo
45-
$(RUN_LAMBDA)/install -d '{}'
46-
@echo
47-
$(RUN_LAMBDA)/install2 -d '{}'
48-
@echo
49-
$(RUN_LAMBDA)/install3 -d '{}'
50-
@echo
51-
@echo
52-
endef
5311

5412
GO = $(abspath ./hack/go.sh)
5513
GO_PATH = hack/go
56-
WORKER_DIR = $(GO_PATH)/src/github.com/open-lambda/open-lambda/worker
5714
ADMIN_DIR = $(GO_PATH)/src/github.com/open-lambda/open-lambda/worker/admin
5815

5916
LAMBDA_DIR = $(abspath ./lambda)
@@ -80,21 +37,15 @@ bin/admin: $(WORKER_GO_FILES)
8037

8138
.PHONY: test-all test-sock-all test-docker-all test-cluster
8239

83-
test-all: test-sock-all test-docker-all
84-
85-
test-sock-all: test-sock-nocache test-sock-handler test-sock-import test-sock-both
40+
test-all: bin/admin imgs/lambda
41+
python -m unittest discover testing/integration-tests/ -p "*_test.py"
8642

87-
test-docker-all: test-docker-nocache test-docker-handler
43+
test-sock-all: bin/admin imgs/lambda
44+
python testing/integration-tests/sock_test.py
8845

89-
test-cluster: imgs/test-cluster
46+
test-docker-all: bin/admin imgs/lambda
47+
python testing/integration-tests/docker_test.py
9048

91-
imgs/test-cluster:
92-
@echo "Starting test cluster..."
93-
./bin/admin new -cluster=$(TEST_CLUSTER)
94-
./bin/admin setconf -cluster=$(TEST_CLUSTER) $(REGISTRY_DIR)
95-
./bin/admin setconf -cluster=$(TEST_CLUSTER) $(STARTUP_PKGS)
96-
@echo
97-
touch imgs/test-cluster
9849

9950
clean-test:
10051
@echo "Killing worker if running..."
@@ -104,30 +55,6 @@ clean-test:
10455
rm -rf $(TEST_CLUSTER) imgs/test-cluster
10556
@echo
10657

107-
test-sock-nocache: bin/admin imgs/lambda test-cluster
108-
$(subst CONDITION, $(SOCK_NOCACHE), $(RUN_TEST))
109-
110-
test-sock-handler: bin/admin imgs/lambda test-cluster
111-
$(subst CONDITION, $(SOCK_HANDLER), $(RUN_TEST))
112-
113-
test-sock-import: bin/admin imgs/lambda test-cluster
114-
$(subst CONDITION, $(SOCK_IMPORT), $(RUN_TEST))
115-
116-
test-sock-both: bin/admin imgs/lambda test-cluster
117-
$(subst CONDITION, $(SOCK_BOTH), $(RUN_TEST))
118-
119-
test-docker-nocache: bin/admin imgs/lambda test-cluster
120-
$(subst CONDITION, $(DOCKER_NOCACHE), $(RUN_TEST))
121-
122-
test-docker-handler: bin/admin imgs/lambda test-cluster
123-
$(subst CONDITION, $(DOCKER_HANDLER), $(RUN_TEST))
124-
125-
test-docker-import: bin/admin imgs/lambda test-cluster
126-
$(subst CONDITION, $(DOCKER_IMPORT), $(RUN_TEST))
127-
128-
test-docker-both: bin/admin imgs/lambda test-cluster
129-
$(subst CONDITION, $(DOCKER_BOTH), $(RUN_TEST))
130-
13158
.PHONY: clean
13259
clean: clean-test
13360
rm -rf bin

‎testing/integration-tests/cluster_test_utils.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
import httplib
2+
import os
3+
import subprocess
4+
import time
5+
6+
def join_paths(base, relative):
7+
return os.path.normpath(os.path.join(base, relative))
8+
9+
INTEGRATION_TESTS_DIR = os.path.dirname(os.path.realpath(__file__))
10+
TEST_CLUSTER_DIR = join_paths(INTEGRATION_TESTS_DIR, '../test-cluster')
11+
REGISTRY_DIR = join_paths(INTEGRATION_TESTS_DIR, '../registry')
12+
ADMIN_BIN = join_paths(INTEGRATION_TESTS_DIR, '../../bin/admin')
13+
WORKER_TIMEOUT = 30
14+
15+
def kill_worker():
16+
template = """{bin} kill -cluster={cluster_dir}; rm -rf {cluster_dir}/workers/*"""
17+
data = { 'bin': ADMIN_BIN, 'cluster_dir': TEST_CLUSTER_DIR }
18+
cmd = template.format(**data)
19+
subprocess.check_output(cmd, shell=True)
20+
21+
def is_worker_active():
22+
template = """{bin} status -cluster={cluster_dir}"""
23+
data = { 'bin': ADMIN_BIN, 'cluster_dir': TEST_CLUSTER_DIR }
24+
cmd = template.format(**data)
25+
devnull = open(os.devnull, 'w')
26+
exit_code = subprocess.call(cmd, shell=True, stdout=devnull)
27+
return exit_code == 0
28+
29+
def set_worker_conf(conf):
30+
template = """{bin} setconf -cluster={cluster_dir} '{conf}'"""
31+
data = { 'bin': ADMIN_BIN, 'cluster_dir': TEST_CLUSTER_DIR, 'conf': conf }
32+
cmd = template.format(**data)
33+
subprocess.check_output(cmd, shell=True)
34+
35+
def start_test_worker():
36+
template = """{bin} workers -cluster={cluster_dir}"""
37+
data = { 'bin': ADMIN_BIN, 'cluster_dir': TEST_CLUSTER_DIR }
38+
cmd = template.format(**data)
39+
subprocess.check_output(cmd, shell=True)
40+
41+
def init_worker(conf):
42+
set_worker_conf(conf)
43+
start_test_worker()
44+
45+
def assert_worker_is_ready():
46+
for i in range(WORKER_TIMEOUT):
47+
time.sleep(2)
48+
if is_worker_active():
49+
return
50+
raise IOError('Worker failed to initialize after '
51+
+ str(WORKER_TIMEOUT * 2) + ' seconds.')
52+
53+
def run_lambda(name):
54+
conn = httplib.HTTPConnection('localhost', 8080, timeout=15)
55+
url = '/runLambda/' + name
56+
conn.request('POST', url, '{}')
57+
response = conn.getresponse()
58+
if response.status != 200:
59+
template = 'Request to run lambda "{name}" failed '
60+
+ 'with status code {status}.'
61+
data = { 'name': name, 'status': response.status }
62+
msg = template.format(**data)
63+
raise IOError(msg)
64+
65+
def create_test_cluster():
66+
template = """{bin} new -cluster={cluster_dir}"""
67+
data = { 'bin': ADMIN_BIN, 'cluster_dir': TEST_CLUSTER_DIR }
68+
cmd = template.format(**data)
69+
subprocess.call(cmd, shell=True)
70+
71+
def start_test_cluster():
72+
print("Starting test cluster...")
73+
create_test_cluster()
74+
75+
STARTUP_PKGS_CONF ='{"startup_pkgs": ["parso", "jedi", "urllib3", "idna", "chardet", "certifi", "requests", "simplejson"]}'
76+
REGISTRY_DIR_CONF ='{"registry_dir": "' + REGISTRY_DIR + '"}'
77+
set_worker_conf(STARTUP_PKGS_CONF)
78+
set_worker_conf(REGISTRY_DIR_CONF)
79+
80+
def run_cluster_test_with_conf(conf):
81+
print('Killing worker if running...')
82+
kill_worker()
83+
print('Starting worker...')
84+
init_worker(conf)
85+
print('Waiting for worker to initialize...')
86+
assert_worker_is_ready()
87+
print('Worker ready. Requesting lambdas...')
88+
run_lambda('echo')
89+
run_lambda('install')
90+
run_lambda('install2')
91+
run_lambda('install3')
92+
93+

‎testing/integration-tests/docker_test.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import json
2+
import unittest
3+
from cluster_test_utils import start_test_cluster, run_cluster_test_with_conf
4+
5+
6+
class DockerTest(unittest.TestCase):
7+
@classmethod
8+
def setUpClass(cls):
9+
start_test_cluster()
10+
11+
def test_with_no_cache(self):
12+
conf = json.dumps({
13+
'sandbox': 'docker',
14+
'handler_cache_size': 0,
15+
'import_cache_size': 0,
16+
'cg_pool_size': 0
17+
})
18+
run_cluster_test_with_conf(conf)
19+
20+
def test_with_handler_cache(self):
21+
conf = json.dumps({
22+
'sandbox': 'docker',
23+
'handler_cache_size': 10000000,
24+
'import_cache_size': 0,
25+
'cg_pool_size': 0
26+
})
27+
run_cluster_test_with_conf(conf)
28+
29+
if __name__ == '__main__':
30+
unittest.main()
31+

‎testing/integration-tests/sock_test.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import json
2+
import unittest
3+
from cluster_test_utils import start_test_cluster, run_cluster_test_with_conf
4+
5+
6+
class SockTest(unittest.TestCase):
7+
@classmethod
8+
def setUpClass(cls):
9+
start_test_cluster()
10+
11+
def test_with_no_cache(self):
12+
conf = json.dumps({
13+
'sandbox': 'sock',
14+
'handler_cache_size': 0,
15+
'import_cache_size': 0,
16+
'cg_pool_size': 10
17+
})
18+
run_cluster_test_with_conf(conf)
19+
20+
def test_with_handle_cache(self):
21+
conf = json.dumps({
22+
'sandbox': 'sock',
23+
'handler_cache_size': 10000000,
24+
'import_cache_size': 0,
25+
'cg_pool_size': 10
26+
})
27+
run_cluster_test_with_conf(conf)
28+
29+
def test_with_import_cache(self):
30+
conf = json.dumps({
31+
'sandbox': 'sock',
32+
'handler_cache_size': 0,
33+
'import_cache_size': 10000000,
34+
'cg_pool_size': 10
35+
})
36+
run_cluster_test_with_conf(conf)
37+
38+
def test_with_both_caches(self):
39+
conf = json.dumps({
40+
'sandbox': 'sock',
41+
'handler_cache_size': 10000000,
42+
'import_cache_size': 10000000,
43+
'cg_pool_size': 10
44+
})
45+
run_cluster_test_with_conf(conf)
46+
47+
if __name__ == '__main__':
48+
unittest.main()
49+

0 commit comments

Comments
 (0)