Skip to content

Commit 9e843ea

Browse files
uujouujo
committed
fix some bugs
1 parent 4771369 commit 9e843ea

11 files changed

Lines changed: 139 additions & 85 deletions

File tree

‎docker-compose.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,22 +32,22 @@ services:
3232
3333
# Initialize S3 in localstack
3434
aws s3api create-bucket --endpoint-url=http://localstack:4572 \
35-
--bucket VARSAP_BUCKET;
35+
--bucket TEST_BUCKET;
3636
3737
# Initialize dynamodb in localstack
3838
aws dynamodb create-table --endpoint-url=http://localstack:4569 \
39-
--table-name ANNOTATION_TABLE \
39+
--table-name TEST_TABLE \
4040
--attribute-definitions \
41-
AttributeName=job_id,AttributeType=S \
41+
AttributeName=key_id,AttributeType=S \
4242
AttributeName=updated,AttributeType=N \
4343
--key-schema \
44-
AttributeName=job_id,KeyType=HASH \
44+
AttributeName=key_id,KeyType=HASH \
4545
--provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5 \
4646
--global-secondary-indexes \
4747
IndexName=SourceIndex,KeySchema=[{AttributeName=updated,KeyType=HASH}],Projection={ProjectionType=KEYS_ONLY},ProvisionedThroughput={ReadCapacityUnits=5,WriteCapacityUnits=5};
4848
4949
# Initialize sqs in localstack
50-
aws sqs create-queue --endpoint-url=http://localstack:4576 --queue-name ANNOTATION_QUEUE;
50+
aws sqs create-queue --endpoint-url=http://localstack:4576 --queue-name TEST_QUEUE;
5151
# you can run more command here
5252
"
5353
networks:

projectname/awsheper/__init__.py renamed to projectname/awshelper/__init__.py

File renamed without changes.

‎projectname/awshelper/awsenv.py‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
"""
2+
This is just the somple code for accessing parameter store and secrets manager
3+
and how to swich local env to aws env
4+
"""
5+
6+
import os
7+
import json
8+
import boto3
9+
10+
LOCAL_TABLE_NAME = "TEST_TABLE"
11+
LOCAL_DYNAMO_ENDPOINT = "http://localstack:4569"
12+
LOCAL_BUCKET_NAME = "TEST_BUCKET"
13+
LOCAL_S3_ENDPOINT = "http://localstack:4572"
14+
LOCAL_QUEUE_NAME = "TEST_QUEUE"
15+
LOCAL_SQS_ENDPOINT = "http://localstack:4576"
16+
17+
18+
def is_local_env():
19+
"""
20+
check whether it is local env which doesn't have real aws set up.
21+
This is set up on template-sam.yaml
22+
:return:
23+
"""
24+
# This is set on template-samp.yaml
25+
return os.getenv("TEST_ENV") == "LOCAL"
26+
27+
28+
def get_params_from_ssm(path_to_config=None):
29+
"""
30+
:param path_to_config: path to the config in Parameter store
31+
:return: dictionary of parameters
32+
"""
33+
34+
config_path = "/need_to_change_to_correct_path"
35+
path_to_config = path_to_config or os.getenv("CONFIG_SSM_PARAMETER", config_path)
36+
37+
ssm = boto3.Session(region_name='us-east-1').client('ssm')
38+
39+
param = ssm.get_parameter(Name=path_to_config, WithDecryption=True)
40+
if param and "Parameter" in param:
41+
param_dict = json.loads(param['Parameter']['Value'])
42+
43+
return param_dict
44+
45+
46+
def get_params_from_secretsmanager(path_to_config=None):
47+
"""
48+
49+
:param path_to_config: secrete location
50+
:return:
51+
"""
52+
client = boto3.Session().client(service_name="secretsmanager", region_name="us-east-1")
53+
response = client.get_secret_value(SecretId=path_to_config)
54+
param_dict = json.loads(response.get('SecretString', "{}"))
55+
56+
return param_dict
57+
58+
59+
@property
60+
def dynamo_endpoint():
61+
"""
62+
returns dynamodb endpoint
63+
"""
64+
return LOCAL_DYNAMO_ENDPOINT if is_local_env() else None
65+
66+
67+
@property
68+
def s3_endpoint():
69+
"""
70+
returns s3 endpoint
71+
"""
72+
return LOCAL_S3_ENDPOINT if is_local_env() else None
73+
74+
75+
@property
76+
def sqs_endpoint():
77+
"""
78+
returns sqs endpoint
79+
"""
80+
return LOCAL_SQS_ENDPOINT if is_local_env() else None
81+
82+
83+
@property
84+
def dynamo_table_name():
85+
"""
86+
returns table name from parameter store
87+
"""
88+
if is_local_env():
89+
return LOCAL_TABLE_NAME
90+
else:
91+
# get data from parameter store with correct key
92+
# table_name = get_params_from_ssm()["CORRECT_KEY"]
93+
return "table_name"
94+
95+
96+
@property
97+
def s3_bucket_name():
98+
"""
99+
returns s3 bucket name from parameter store
100+
"""
101+
if is_local_env():
102+
return LOCAL_BUCKET_NAME
103+
else:
104+
# get data from parameter store with correct key
105+
# bucket_name = get_params_from_ssm()["CORRECT_KEY"]
106+
return "bucket_name"
107+
108+
109+
@property
110+
def sqs_name():
111+
"""
112+
returns sqs name from parameter store
113+
"""
114+
if is_local_env():
115+
return LOCAL_QUEUE_NAME
116+
else:
117+
# get data from parameter store with correct key
118+
# sqs_name = get_params_from_ssm()["CORRECT_KEY"]
119+
return "sqs_name"

‎projectname/awsheper/awsenv.py‎

Lines changed: 0 additions & 56 deletions
This file was deleted.

‎projectname/handler/lambda_handler_template.py‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,35 @@
44
It is just a sample. It doesn't include proper error handling codes
55
66
"""
7-
import os
87
import json
98
import logging
10-
import boto3
119
import time
1210

11+
import boto3
12+
13+
from ..awshelper.awsenv import dynamo_endpoint, s3_endpoint, dynamo_table_name, s3_bucket_name
1314

14-
def get_s3_data(bucket_name, key):
15+
16+
def get_s3_data(key):
1517
"""
1618
getting s3 data
1719
"""
18-
endpoint_url = os.environ.get("LOCAL_S3_ENDPOINT")
19-
s3 = boto3.resource("s3", endpoint_url=endpoint_url)
20-
s3_obj = s3.Object(bucket_name, key=key)
20+
s3 = boto3.resource("s3", endpoint_url=s3_endpoint)
21+
s3_obj = s3.Object(s3_bucket_name, key=key)
2122
data = s3_obj.get(ResponseContentType="application/json")
2223
body = json.loads(data["Body"].read())
2324
return body
2425

2526

26-
def inset_to_dynamo(key, data):
27+
def insert_to_dynamo(key, data):
2728
"""
2829
insert data to dynamodb
2930
:param key:
3031
:param data:
3132
:return:
3233
"""
33-
endpoint_url = os.environ.get("LOCAL_DYNAMODB")
34-
dynamo = boto3.resource("dynamodb", endpoint_url=endpoint_url)
35-
table_name = os.environ.get("TEST_TABLE_NAME")
36-
dynamo_table = dynamo.Table(table_name)
34+
dynamo = boto3.resource("dynamodb", endpoint_url=dynamo_endpoint)
35+
dynamo_table = dynamo.Table(dynamo_table_name)
3736

3837
item = {
3938
"key_id": key,
@@ -45,7 +44,7 @@ def inset_to_dynamo(key, data):
4544
return result
4645

4746

48-
def populate_dyanmo(event, context):
47+
def populate_dynamo(event, context):
4948
"""
5049
:param event: aws event
5150
:param context: aws context
@@ -61,7 +60,5 @@ def populate_dyanmo(event, context):
6160
if record.get("eventSource") == "aws:sqs" and "body" in record:
6261
sqs_data = json.loads(record["body"])
6362
# assume sqs contains s3 bucket name and key
64-
data = get_s3_data(sqs_data["bucket_name"], sqs_data["key"])
65-
inset_to_dynamo(sqs_data["key"], data)
66-
67-
63+
data = get_s3_data(sqs_data["key"])
64+
insert_to_dynamo(sqs_data["key"], data)

‎requirements.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ pytest
66
pytest-cov
77
black
88
pyinstrument
9+
pylint

bin/start_local_test_template.sh renamed to start_local_test_template.sh

File renamed without changes.

bin/stop_local_test_template.sh renamed to stop_local_test_template.sh

File renamed without changes.

‎template-sam.yaml‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,10 @@ Globals:
88
Environment:
99
Variables:
1010
# Set localtest env
11-
TEST_ENV: LOCAL_TEST
1211
# Set environment variable so amazon service can be switched in local settings
1312
# This has to be matched with localstack config in docker-compose.yaml
1413
# Current example contains DynamoDB and S3
15-
ANNOTATION_TABLE: ANNOTATION_TABLE
16-
LOCAL_DYNAMODB: http://localstack:4569
17-
BUCKET_NAME: VARSAP_BUCKET
18-
LOCAL_S3_ENDPOINT: http://localstack:4572
19-
ANNOTATION_QUEUE: ANNOTATION_QUEUE
20-
LOCAL_SQS_ENDPOINT: http://localstack:4576
21-
LOG_LEVEL: DEBUG
14+
TEST_ENV: LOCAL
2215

2316
Resources:
2417
RequestAnnotation:

‎tests/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)