Skip to content

Commit 8970c8d

Browse files
uujouujo
authored
Merge pull request #4 from uujo/dynamo_tests
add dynamo test
2 parents 7c2e7f0 + da0ef6f commit 8970c8d

38 files changed

Lines changed: 760 additions & 130 deletions

‎.git_templates/hooks/pre-commit‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,4 @@ black --config ./pyproject.toml .
1414

1515
# Run pylint and check the score
1616
echo "pre-commit: Running lint check"
17-
pylint --rcfile=./pylintrc projectname/ tests/
17+
pylint --rcfile=./pylintrc src/projectname tests/unittests

‎.git_templates/hooks/pre-push‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
# run the unittests if it fails abort the push
1010
echo "pre-push: Run unittest"
11-
coverage run --source=projectname -m pytest tests/unittests/
11+
coverage run --source=src/projectname -m pytest tests/unittests/
1212

1313
# Above command doesn't exit when unittests fails
1414
# It needs to be caught and exit non 0 to abort the push

‎.travis.yml‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
dist: xenial
2+
language: python
3+
4+
branches:
5+
only: master
6+
7+
sudo: true
8+
9+
python: 3.7
10+
11+
before_install:
12+
- export AUTHOR=`git --no-pager show -s --format='%an <%ae>'`
13+
- export DATE=`TZ=America/New_York date "+%m-%d-%y-%H%M"`
14+
- export DATE_TM=`TZ=America/New_York date "+%m-%d-%y %H:%M"`
15+
- python --version
16+
17+
env:
18+
global:
19+
- AWS_DEFAULT_REGION=us-east-1
20+
- PYTHONPATH=$TRAVIS_BUILD_DIR:$PYTHONPATH
21+
22+
install:
23+
- pip install awscli --upgrade
24+
- pip install -r requirements-travis.txt
25+
- pip install awscli --upgrade
26+
27+
script:
28+
# run tests
29+
#- py.test -vv -r sxX
30+
- export PYTHONPATH=$PWD/src:$PWD/tests:$PYTHONPATH
31+
- BOTO_CONFIG=none pytest --cov-report xml --cov=src/projectname/ tests/unittests/
32+
- coverage xml && python-codacy-coverage -r coverage.xml
33+
- ls -alh
34+
35+
- make
36+
# consult cloud engineer for set up below
37+
#- mv ./dist/lambda.zip ./dist/[lamdba-name]:latest.zip
38+
#- cp ./dist/[lamdba-name]:latest.zip ./dist/[lambda-name]:$DATE.zip
39+
#- cp ./dist/[lamdba-name]:latest.zip ./dist/[lambda-name]:latest.zip
40+
- ls -alh ./dist/
41+
42+
43+
after_success:
44+
45+
# consult a cloud engineer for set up below
46+
#- aws s3 cp ../dist/[lamdba-name]:$DATE.zip s3://biad-lambda-hub/[ProjectName]/ --acl public-read
47+
# run integration test if exists
48+
#- pytest -vv ./tests/integrationtests/ || exit $?
49+
50+
# consult a cloud engineer for heartbeat test
51+
52+
notifications:
53+
# consult a cloud engineer for slack notification
54+
#slack: clinicalbiomed:EphqNgtuf0NzIGaOE43vSFDc

‎Makefile‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
default: clean install copy zip
2+
3+
install: build_path
4+
pip install -r requirements-lambda.txt -t ./build
5+
6+
build_path:
7+
mkdir ./build
8+
9+
build_dist:
10+
mkdir ./dist
11+
12+
copy:
13+
ls -alh
14+
cp -R src/* ./build/
15+
ls -alh ./build
16+
17+
zip: build_dist
18+
cd ./build && zip -r9 ./dist/lambda.zip . 2>&1 >>/dev/null
19+
20+
clean:
21+
rm -rf ./build
22+
rm -rf ./dist

‎README.md‎

Lines changed: 32 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,23 @@
1-
# lambda_localstack_template
2-
Template for lambda with localstack and sam integration for localtest
1+
# nci-aws-lambda-localstack-template
2+
3+
This provides a templates for aws lambda test in local environment using localstack or local server instances (i.e. dynamoDB). This examples shows only fraction of the services localstack provides. The full list of services are [here](https://github.com/localstack/localstack). This template provides the basic settings for local test which can be used in different projects with minimal configuration changes. Details for the configuration changes is explained `How to use this template` section below.
34

45
## Prerequesite (on Mac)
56

67
* [Python3](https://www.python.org/downloads/)
78

8-
* [AWS client](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) (pip3 install awscli --upgrade --user)
9+
* [Docker](https://docs.docker.com/docker-for-mac/install/) - need for the local lambda function testing with sam
910

10-
* [AWS SAM client](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install-mac.html#serverless-sam-cli-install-mac-pip) (pip3 install --user aws-sam-cli)
11-
12-
* Reference: [Adjust path to use sam cli command](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install-mac-path.html)
11+
* [AWS client](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
1312

14-
* [Docker](https://docs.docker.com/docker-for-mac/install/) - need for the local lambda function testing
13+
* [AWS SAM client](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install-mac.html)
14+
1515

1616
## getting the source code
1717

18-
`git clone https://github.com/uujo/lambda_localstack_template.git`
18+
`git clone https://github.com/BIAD/nci-aws-lambda-localstack-template.git`
1919

20-
`cd lambda_localstack_template`
20+
`cd nci-aws-lambda-localstack-template`
2121

2222

2323
## Set up git hooks (pre-commit, pre-push) - This is one time setting. Doesn't have to be repeated every time.
@@ -39,67 +39,50 @@ For special cases, if you still need to commit, push the code use _--no-verify_
3939
> git push origin [branch_name] --no-verify
4040
```
4141

42-
## Local test set up using aws sam and localstack (with python3 venv)
42+
## set PYTHONPATH for unittest
43+
44+
`export PYTHONPATH=$PWD/src:$PWD/tests:$PYTHONPATH`
45+
46+
If virtual environment is used below, you can set the above path in __.venv/bin/activate__ file
47+
48+
49+
## Local test set up using aws sam and localstack (with python3 venv) - virtual environment setting is optional.
4350

4451
* set the virtual environment (optional, you can use other virtual env tools or without it)
4552

46-
`cd lambda_localstack_template`
53+
`cd nci-aws-lambda-localstack-template`
4754

4855
`python3 -m venv .venv`
4956

5057
`source .venv/bin/activate` - activate virtual environment
5158

5259
`pip install -r requirements.txt` - one time setting
5360
(If install fails with __xcun__ error, run __xcode-select --install__)
54-
55-
56-
* To start local test setup
61+
62+
* Now it is ready to set up the test below.
63+
64+
* After all the test is done. get out of virtual environment.
5765

58-
`./start_local_test_template.sh`
66+
`deactivate`
5967

60-
_-i_ option sets interactive mode on localstack instead of running it background, if you want to monitor localstack log use this option. Without this option, it automatically create the sample data in S3 bucket.
61-
68+
## Tests Examples
6269

63-
* To check the table, sqs and bucket are created (**Use this setting only if you use _-i_ option, without _-i_, these steps runs automatically**)
64-
65-
`aws dynamodb list-tables --endpoint-url http://localhost:4569`
66-
67-
`aws sqs list-queues --endpoint-url=http://localhost:4576`
68-
69-
`aws s3api list-buckets --endpoint-url=http://localhost:4572`
70-
71-
* put data in S3
72-
73-
`aws s3api put-object --endpoint-url=http://localhost:4572 --bucket TEST_BUCKET --key test_path/data.json --body tests/s3_test_data.json`
70+
### [Lambda with dynamo, API gateway testing (Localstack)](https://github.com/BIAD/nci-aws-lambda-localstack-template/wiki/Lambda-Dynamo-Test)
7471

75-
* To check S3 object is created
72+
73+
### [Lambda with dynamo, s3, sqs testing (Localstack)](https://github.com/BIAD/nci-aws-lambda-localstack-template/wiki/Lambda-S3-SQS-Dynamo-Test)
7674

77-
`aws s3api list-objects --bucket=TEST_BUCKET --endpoint-url=http://localhost:4572`
78-
79-
80-
* To test lambdas
75+
### [Lambda with dynamo, API gateway testing (local dynamo server)](https://github.com/BIAD/nci-aws-lambda-localstack-template/wiki/Lambda-Local-Dynamo-Server-Test)
8176

82-
* invoke SQS event
83-
84-
`sam local invoke TestLambda -t build/template.yaml --docker-network lambda_localstack_template_local_aws_network -e tests/sqs_test_event.json`
8577

86-
* To check whether table has an entry inserted.
78+
## Integration Test: To be added
8779

88-
`aws dynamodb scan --table-name TEST_TABLE --endpoint-url http://localhost:4569`
89-
90-
* To check sqs messge is queued
91-
92-
`aws sqs receive-message --queue-url http://localhost:4576/queue/TEST_QUEUE --endpoint-url=http://localhost:4576 --max-number-of-messages=10`
93-
94-
95-
* To stop the local test and clean up docker
9680

97-
`./stop_local_test_template.sh`
98-
81+
## [How to use this template](https://github.com/BIAD/nci-aws-lambda-localstack-template/wiki/How-to-use-this-template)
9982

100-
## Integration Test: To be added
83+
* goal: With minimal configuation change, testing environment can be set up for different project.
84+
* There are two main configuration - One for localstack and one for sam. If starting and stop scrip need to be changed as well if you want to reuse the script.
10185

102-
10386
## Reference
10487
* [AWS SAM guick start guide](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-quick-start.html)
10588
* [SAM template](https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md)

‎local-test-configs/apigw-dynamo-config/api-gw-template-sam.yaml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: AWS::Serverless-2016-10-31
3+
4+
Globals:
5+
Function:
6+
Runtime: python3.7
7+
Timeout: 60
8+
Environment:
9+
Variables:
10+
# Set localtest env
11+
# Set environment variable so amazon service can be switched in local settings
12+
# This has to be matched with localstack config in docker-compose.yaml
13+
# Current example contains DynamoDB
14+
TEST_ENV: LOCAL
15+
16+
Resources:
17+
PostTest:
18+
Type: AWS::Serverless::Function # More info about Function Resource: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#awsserverlessfunction
19+
Properties:
20+
CodeUri: .
21+
Handler: projectname.handler.dynamo_handler.post_handler.post
22+
Events:
23+
Validation:
24+
Type: Api # More info about API Event Source: https://github.com/awslabs/serverless-application-model/blob/master/versions/2016-10-31.md#api
25+
Properties:
26+
Path: /api/v1/testapi
27+
Method: post
28+
29+
GetTest:
30+
Type: AWS::Serverless::Function
31+
Properties:
32+
CodeUri: .
33+
Handler: projectname.handler.dynamo_handler.get_handler.get
34+
Events:
35+
Annotation:
36+
Type: Api
37+
Properties:
38+
Path: /api/v1/testapi/{id}
39+
Method: get
40+
41+
PutTest:
42+
Type: AWS::Serverless::Function
43+
Properties:
44+
CodeUri: .
45+
Handler: projectname.handler.dynamo_handler.put_handler.put
46+
Events:
47+
Annotation:
48+
Type: Api
49+
Properties:
50+
Path: /api/v1/testapi/{id}
51+
Method: put
52+
53+
DeleteTest:
54+
Type: AWS::Serverless::Function
55+
Properties:
56+
CodeUri: .
57+
Handler: projectname.handler.dynamo_handler.delete_handler.delete
58+
Events:
59+
Annotation:
60+
Type: Api
61+
Properties:
62+
Path: /api/v1/testapi/{id}
63+
Method: delete

‎local-test-configs/apigw-dynamo-config/docker-compose.yaml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
version: "3"
2+
3+
services:
4+
localstack:
5+
image: localstack/localstack:0.11.1
6+
ports:
7+
- "4566-4599:4566-4599"
8+
- "${PORT_WEB_UI-8080}:${PORT_WEB_UI-8080}"
9+
environment:
10+
- DOCKER_HOST=unix:///var/run/docker.sock
11+
- SERVICES=dynamodb
12+
- DEFAULT_REGION=us-east-1
13+
- EDGE_PORT=4566
14+
volumes:
15+
- /var/run/docker.sock:/var/run/docker.sock
16+
- /private${TMPDIR}:/tmp/localstack
17+
networks:
18+
- local_aws_network
19+
20+
aws-cli:
21+
image: mesosphere/aws-cli
22+
volumes:
23+
- ./dev_env:/project/dev_env
24+
environment:
25+
- AWS_ACCESS_KEY_ID=1234
26+
- AWS_SECRET_ACCESS_KEY=1234
27+
- AWS_DEFAULT_REGION=us-east-1
28+
entrypoint: /bin/sh -c
29+
command: >
30+
"
31+
# Sleep is needed so all localstack components will startup correctly (There's a better way to do this)
32+
sleep 10;
33+
34+
# Initialize dynamodb in localstack
35+
aws dynamodb create-table --endpoint-url=http://localstack:4566 \
36+
--table-name TEST_TABLE \
37+
--attribute-definitions \
38+
AttributeName=id,AttributeType=S \
39+
--key-schema \
40+
AttributeName=id,KeyType=HASH \
41+
--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5;
42+
43+
# you can run more command here
44+
"
45+
networks:
46+
- local_aws_network
47+
depends_on:
48+
- localstack
49+
50+
networks:
51+
local_aws_network:

0 commit comments

Comments
 (0)