Skip to content

Commit b8d8d8b

Browse files
whummerwhummer
authored
fix chunked encoding for S3 uploads; refactor GenericProxyHandler class (localstack#416)
1 parent bd92399 commit b8d8d8b

12 files changed

Lines changed: 36 additions & 25 deletions

File tree

‎.travis.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ script:
4646
- make init
4747
- DEBUG=1 LAMBDA_EXECUTOR=docker make test
4848
- LAMBDA_EXECUTOR=local USE_SSL=1 make test
49+
# run Java tests
50+
- make test-java-if-changed
4951
# build Docker image, and push it (if on master branch)
5052
- make docker-build
5153
- make docker-push-master

‎Makefile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ docker-push-master:## Push Docker image to registry IF we are currently on the m
6666
BASE_IMAGE_ID=`docker history -q $(IMAGE_NAME):$(IMAGE_TAG) | tail -n 1`; \
6767
docker-squash -t $(IMAGE_NAME):$(IMAGE_TAG) -f $$BASE_IMAGE_ID $(IMAGE_NAME):$(IMAGE_TAG) && \
6868
docker tag $(IMAGE_NAME):$(IMAGE_TAG) $(IMAGE_NAME):latest; \
69-
make test-java-docker && \
7069
((! (git diff HEAD~1 localstack/constants.py | grep '^+VERSION =') && echo "Only pushing tag 'latest' as version has not changed.") || \
7170
docker push $(IMAGE_NAME):$(IMAGE_TAG)) && \
7271
docker push $(IMAGE_NAME):latest \
@@ -88,6 +87,9 @@ test: ## Run automated tests
8887
test-java: ## Run tests for Java/JUnit compatibility
8988
cd localstack/ext/java; mvn -q test && USE_SSL=1 mvn -q test
9089

90+
test-java-if-changed:
91+
@(! (git log -n 1 --no-merges --raw | grep localstack/ext/java/)) || make test-java
92+
9193
test-java-docker:
9294
ENTRYPOINT="--entrypoint=" CMD="make test-java" make docker-run
9395

‎localstack/services/apigateway/apigateway_listener.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,7 @@ def get_resource_for_path(path, path_map):
127127
class ProxyListenerApiGateway(ProxyListener):
128128

129129
def forward_request(self, method, path, data, headers):
130+
data = data and json.loads(data)
130131

131132
# Paths to match
132133
regex2 = r'^/restapis/([A-Za-z0-9_\-]+)/([A-Za-z0-9_\-]+)/%s/(.*)$' % PATH_USER_REQUEST

‎localstack/services/cloudformation/cloudformation_listener.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
from requests.models import Response, Request
77
from six.moves.urllib import parse as urlparse
88
from localstack.constants import DEFAULT_REGION, TEST_AWS_ACCOUNT_ID
9+
from localstack.utils.common import to_str
910
from localstack.utils.aws import aws_stack
1011
from localstack.utils.cloudformation import template_deployer
1112
from localstack.services.generic_proxy import ProxyListener
@@ -129,7 +130,7 @@ class ProxyListenerCloudFormation(ProxyListener):
129130
def forward_request(self, method, path, data, headers):
130131
req_data = None
131132
if method == 'POST' and path == '/':
132-
req_data = urlparse.parse_qs(data)
133+
req_data = urlparse.parse_qs(to_str(data))
133134
action = req_data.get('Action')[0]
134135

135136
if req_data:
@@ -154,7 +155,7 @@ def forward_request(self, method, path, data, headers):
154155
def return_response(self, method, path, data, headers, response):
155156
req_data = None
156157
if method == 'POST' and path == '/':
157-
req_data = urlparse.parse_qs(data)
158+
req_data = urlparse.parse_qs(to_str(data))
158159
action = req_data.get('Action')[0]
159160

160161
if req_data:

‎localstack/services/dynamodb/dynamodb_listener.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,15 @@
2626
class ProxyListenerDynamoDB(ProxyListener):
2727

2828
def forward_request(self, method, path, data, headers):
29+
data = json.loads(data)
30+
2931
if random.random() < config.DYNAMODB_ERROR_PROBABILITY:
3032
return error_response_throughput()
3133
return True
3234

3335
def return_response(self, method, path, data, headers, response):
36+
data = json.loads(data)
37+
3438
# update table definitions
3539
if data and 'TableName' in data and 'KeySchema' in data:
3640
TABLE_DEFINITIONS[data['TableName']] = data

‎localstack/services/generic_proxy.py‎

Lines changed: 4 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,16 @@
22
import requests
33
import os
44
import sys
5-
import json
65
import traceback
76
import logging
87
import ssl
98
from flask_cors import CORS
109
from requests.structures import CaseInsensitiveDict
1110
from requests.models import Response, Request
12-
from six import iteritems, string_types
11+
from six import iteritems
1312
from six.moves.socketserver import ThreadingMixIn
1413
from six.moves.urllib.parse import urlparse
15-
from localstack.config import DEFAULT_ENCODING, TMP_FOLDER, USE_SSL
14+
from localstack.config import TMP_FOLDER, USE_SSL
1615
from localstack.constants import ENV_INTERNAL_TEST_RUN
1716
from localstack.utils.common import FuncThread, generate_ssl_cert, to_bytes
1817

@@ -144,16 +143,7 @@ def forward(self, method):
144143
target_url = self.path
145144
if '://' not in target_url:
146145
target_url = '%s%s' % (self.proxy.forward_url, target_url)
147-
data = None
148-
if method in ['POST', 'PUT', 'PATCH']:
149-
data_string = self.data_bytes
150-
try:
151-
if not isinstance(data_string, string_types):
152-
data_string = data_string.decode(DEFAULT_ENCODING)
153-
data = json.loads(data_string)
154-
except Exception as e:
155-
# unable to parse JSON, fallback to verbatim string/bytes
156-
data = data_string
146+
data = self.data_bytes
157147

158148
forward_headers = CaseInsensitiveDict(self.headers)
159149
# update original "Host" header (moto s3 relies on this behavior)
@@ -181,6 +171,7 @@ def forward(self, method):
181171
self.send_response(code)
182172
self.end_headers()
183173
return
174+
# perform the actual invocation of the backend service
184175
if response is None:
185176
if modified_request:
186177
response = self.method(proxy_url, data=modified_request.data,

‎localstack/services/kinesis/kinesis_listener.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,15 @@
1818
class ProxyListenerKinesis(ProxyListener):
1919

2020
def forward_request(self, method, path, data, headers):
21+
data = json.loads(data)
22+
2123
if random.random() < config.KINESIS_ERROR_PROBABILITY:
2224
return kinesis_error_response(data)
2325
return True
2426

2527
def return_response(self, method, path, data, headers, response):
2628
action = headers.get('X-Amz-Target')
29+
data = json.loads(data)
2730

2831
records = []
2932
if action in (ACTION_CREATE_STREAM, ACTION_DELETE_STREAM):

‎localstack/services/s3/s3_listener.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,14 +207,15 @@ def strip_chunk_signatures(data):
207207
# <payload data ...>
208208
# 0;chunk-signature=927ab45acd82fc90a3c210ca7314d59fedc77ce0c914d79095f8cc9563cf2c70
209209

210-
data_new = re.sub(r'^[0-9a-fA-F]+;chunk-signature=[0-9a-f]{64}', '', data, flags=re.MULTILINE)
210+
data_new = re.sub(b'(\r\n)?[0-9a-fA-F]+;chunk-signature=[0-9a-f]{64}(\r\n){,2}', b'',
211+
data, flags=re.MULTILINE | re.DOTALL)
211212
if data_new != data:
212213
# trim \r (13) or \n (10)
213214
for i in range(0, 2):
214-
if ord(data_new[0]) in (10, 13):
215+
if data_new[0] in (10, 13):
215216
data_new = data_new[1:]
216217
for i in range(0, 6):
217-
if ord(data_new[-1]) in (10, 13):
218+
if data_new[-1] in (10, 13):
218219
data_new = data_new[:-1]
219220
return data_new
220221

‎localstack/services/sns/sns_listener.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from requests.models import Response
66
from six.moves.urllib import parse as urlparse
77
from localstack.utils.aws import aws_stack
8-
from localstack.utils.common import short_uid
8+
from localstack.utils.common import short_uid, to_str
99
from localstack.services.awslambda import lambda_api
1010
from localstack.services.generic_proxy import ProxyListener
1111

@@ -21,7 +21,7 @@ class ProxyListenerSNS(ProxyListener):
2121
def forward_request(self, method, path, data, headers):
2222

2323
if method == 'POST' and path == '/':
24-
req_data = urlparse.parse_qs(data)
24+
req_data = urlparse.parse_qs(to_str(data))
2525
req_action = req_data['Action'][0]
2626
topic_arn = req_data.get('TargetArn') or req_data.get('TopicArn')
2727

@@ -96,7 +96,7 @@ def return_response(self, method, path, data, headers, response):
9696
# This method is executed by the proxy after we've already received a
9797
# response from the backend, hence we can utilize the "reponse" variable here
9898
if method == 'POST' and path == '/':
99-
req_data = urlparse.parse_qs(data)
99+
req_data = urlparse.parse_qs(to_str(data))
100100
req_action = req_data['Action'][0]
101101
if req_action == 'Subscribe' and response.status_code < 400:
102102
response_data = xmltodict.parse(response.content)

‎localstack/services/sqs/sqs_listener.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ class ProxyListenerSQS(ProxyListener):
1515
def forward_request(self, method, path, data, headers):
1616

1717
if method == 'POST' and path == '/':
18-
req_data = urlparse.parse_qs(data)
18+
req_data = urlparse.parse_qs(to_str(data))
1919
if 'QueueName' in req_data:
2020
if '.' in req_data['QueueName'][0]:
2121
# ElasticMQ currently does not support "." in the queue name, e.g., for *.fifo queues
@@ -30,7 +30,7 @@ def forward_request(self, method, path, data, headers):
3030
def return_response(self, method, path, data, headers, response):
3131

3232
if method == 'POST' and path == '/':
33-
req_data = urlparse.parse_qs(data)
33+
req_data = urlparse.parse_qs(to_str(data))
3434
action = req_data.get('Action', [None])[0]
3535
event_type = None
3636
queue_url = None

0 commit comments

Comments
 (0)