Skip to content

Commit fee78c5

Browse files
bugybugy
committed
bugy#184 changed REST script start request to streaming, to upload huge files efficiently + added config for max http request size
1 parent 13eb13d commit fee78c5

16 files changed

Lines changed: 891 additions & 47 deletions

‎.travis.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ before_install:
2222
- sudo apt-get -y install python3-pip python3-setuptools
2323
install:
2424
- sudo pip3 install -r requirements.txt
25-
- sudo pip3 install ldap3
25+
- sudo pip3 install ldap3 requests parameterized
2626
- sudo pip3 install pyasn1 --upgrade
2727
- cd web-src
2828
- npm install

‎samples/configs/parameterized.json‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
"name": "Very parameterized",
33
"script_path": "scripts/parameterized.sh",
44
"description": "This script does nothing except accepting a lot of parameters and printing them",
5-
"requires_terminal": false,
65
"allowed_users": [
76
"*"
87
],

‎samples/scripts/parameterized.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,15 @@ printf '%s\n' "${args[@]}"
3737
echo
3838

3939
if [ ! -z "$recurs_file" ]; then
40-
echo "recurs_file=$recurs_file"
41-
echo "md5="`md5sum "$recurs_file"`
40+
echo "recurs_file="`md5sum "$recurs_file"`
4241
fi
4342

4443
echo
4544

4645
if [ -z "$my_file" ]; then
4746
echo '--file_upload is empty'
4847
else
49-
echo '--file_upload content:'
50-
cat "$my_file"
48+
echo "--file_upload: "`md5sum "$my_file"`
5149
fi
5250

5351
sleep 5

‎src/features/file_upload_feature.py‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import os
33

44
from files.user_file_storage import UserFileStorage
5-
from utils import file_utils
5+
from utils.file_utils import normalize_path
66

77
RESULT_FILES_FOLDER = 'uploadFiles'
88

@@ -16,11 +16,6 @@ def __init__(self, user_file_storage: UserFileStorage, temp_folder) -> None:
1616

1717
user_file_storage.start_autoclean(self.folder, 1000 * 60 * 60 * 24)
1818

19-
def save_file(self, filename, body, username) -> str:
20-
upload_folder = self.user_file_storage.prepare_new_folder(username, self.folder)
21-
pref_result_path = os.path.join(upload_folder, filename)
22-
23-
result_path = file_utils.create_unique_filename(pref_result_path)
24-
file_utils.write_file(result_path, body, True)
25-
26-
return file_utils.normalize_path(result_path)
19+
def prepare_new_folder(self, username) -> str:
20+
new_folder = self.user_file_storage.prepare_new_folder(username, self.folder)
21+
return normalize_path(new_folder)

‎src/model/model_helper.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,25 @@ def read_bool(value):
115115
return value.lower() == 'true'
116116

117117

118+
def read_int_from_config(key, config_obj, *, default=None):
119+
value = config_obj.get(key)
120+
if value is None:
121+
return default
122+
123+
if isinstance(value, int) and not isinstance(value, bool):
124+
return value
125+
126+
if isinstance(value, str):
127+
if value.strip() == '':
128+
return default
129+
try:
130+
return int(value)
131+
except ValueError as e:
132+
raise InvalidValueException(key, 'Invalid %s value: integer expected, but was: %s' % (key, value)) from e
133+
134+
raise InvalidValueTypeException('Invalid %s value: integer expected, but was: %s' % (key, repr(value)))
135+
136+
118137
def is_empty(value):
119138
return (not value) and (value != 0) and (value is not False)
120139

@@ -197,3 +216,8 @@ class InvalidValueException(Exception):
197216
def __init__(self, param_name, validation_error) -> None:
198217
super().__init__(validation_error)
199218
self.param_name = param_name
219+
220+
221+
class InvalidValueTypeException(Exception):
222+
def __init__(self, message) -> None:
223+
super().__init__(message)

‎src/model/server_conf.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import utils.file_utils as file_utils
66
from auth.authorization import ANY_USER
77
from model import model_helper
8-
from model.model_helper import read_list
8+
from model.model_helper import read_list, read_int_from_config
99
from utils.string_utils import strip
1010

1111
LOGGER = logging.getLogger('server_conf')
@@ -26,6 +26,7 @@ def __init__(self) -> None:
2626
self.trusted_ips = []
2727
self.user_groups = None
2828
self.admin_users = []
29+
self.max_request_size_mb = None
2930

3031
def get_port(self):
3132
return self.port
@@ -129,6 +130,8 @@ def from_json(conf_path, temp_folder):
129130
config.user_groups = user_groups
130131
config.admin_users = admin_users
131132

133+
config.max_request_size_mb = read_int_from_config('max_request_size', json_object, default=10)
134+
132135
return config
133136

134137

‎src/tests/file_upload_feature_test.py‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import os
2+
import time
23
import unittest
34

45
from features.file_upload_feature import FileUploadFeature
@@ -17,16 +18,17 @@ def tearDown(self):
1718
test_utils.cleanup()
1819
self.__storage._stop_autoclean()
1920

20-
def test_create_file(self):
21-
file_path = self.upload_feature.save_file('my_file.txt', b'test text', 'userX')
21+
def test_prepare_new_folder(self):
22+
file_path = self.upload_feature.prepare_new_folder('userX')
2223
self.assertTrue(os.path.exists(file_path))
2324

24-
def test_content_in_created_file(self):
25-
file_path = self.upload_feature.save_file('my_file.txt', b'My text', 'userX')
26-
content = file_utils.read_file(file_path)
27-
self.assertEqual('My text', content)
25+
def test_prepare_new_folder_different_users(self):
26+
path1 = self.upload_feature.prepare_new_folder('userX')
27+
path2 = self.upload_feature.prepare_new_folder('userY')
28+
self.assertNotEqual(path1, path2)
2829

29-
def test_same_filename(self):
30-
file_path1 = self.upload_feature.save_file('my_file.txt', b'some text', 'userX')
31-
file_path2 = self.upload_feature.save_file('my_file.txt', b'some text', 'userX')
30+
def test_prepare_new_folder_twice(self):
31+
file_path1 = self.upload_feature.prepare_new_folder('userX')
32+
time.sleep(0.1)
33+
file_path2 = self.upload_feature.prepare_new_folder('userX')
3234
self.assertNotEqual(file_path1, file_path2)

‎src/tests/model_helper_test.py‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
from config.constants import FILE_TYPE_FILE, FILE_TYPE_DIR
55
from model import model_helper
66
from model.model_helper import read_list, read_dict, fill_parameter_values, resolve_env_vars, \
7-
InvalidFileException, read_bool_from_config
7+
InvalidFileException, read_bool_from_config, InvalidValueException, InvalidValueTypeException
88
from tests import test_utils
99
from tests.test_utils import create_parameter_model, set_env_value
1010

@@ -282,3 +282,35 @@ def setUp(self):
282282

283283
def tearDown(self):
284284
test_utils.cleanup()
285+
286+
287+
class TestReadIntFromConfig(unittest.TestCase):
288+
def test_normal_int_value(self):
289+
value = model_helper.read_int_from_config('abc', {'abc': 123})
290+
self.assertEqual(123, value)
291+
292+
def test_zero_int_value(self):
293+
value = model_helper.read_int_from_config('abc', {'abc': 0})
294+
self.assertEqual(0, value)
295+
296+
def test_string_value(self):
297+
value = model_helper.read_int_from_config('abc', {'abc': '-666'})
298+
self.assertEqual(-666, value)
299+
300+
def test_string_value_when_invalid(self):
301+
self.assertRaises(InvalidValueException, model_helper.read_int_from_config, 'abc', {'abc': '1000b'})
302+
303+
def test_unsupported_type(self):
304+
self.assertRaises(InvalidValueTypeException, model_helper.read_int_from_config, 'abc', {'abc': True})
305+
306+
def test_default_value(self):
307+
value = model_helper.read_int_from_config('my_key', {'abc': 100})
308+
self.assertIsNone(value)
309+
310+
def test_default_value_explicit(self):
311+
value = model_helper.read_int_from_config('my_key', {'abc': 100}, default=5)
312+
self.assertEqual(5, value)
313+
314+
def test_default_value_when_empty_string(self):
315+
value = model_helper.read_int_from_config('my_key', {'my_key': ' '}, default=9999)
316+
self.assertEqual(9999, value)

‎src/tests/server_conf_test.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,20 @@ def tearDown(self):
105105
test_utils.cleanup()
106106

107107

108+
class TestMaxRequestSize(unittest.TestCase):
109+
def test_int_value(self):
110+
config = _from_json({'max_request_size': 5})
111+
self.assertEqual(5, config.max_request_size_mb)
112+
113+
def test_string_value(self):
114+
config = _from_json({'max_request_size': '123'})
115+
self.assertEqual(123, config.max_request_size_mb)
116+
117+
def test_default_value(self):
118+
config = _from_json({})
119+
self.assertEqual(10, config.max_request_size_mb)
120+
121+
108122
def _from_json(content):
109123
json_obj = json.dumps(content)
110124
conf_path = os.path.join(test_utils.temp_folder, 'conf.json')

‎src/tests/test_utils.py‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import stat
55
import threading
66
import uuid
7+
from collections import Set
78

89
import utils.file_utils as file_utils
910
import utils.os_utils as os_utils
@@ -15,6 +16,7 @@
1516
temp_folder = 'tests_temp'
1617
_original_env = {}
1718

19+
1820
def create_file(filepath, overwrite=False):
1921
if not os.path.exists(temp_folder):
2022
os.makedirs(temp_folder)
@@ -54,14 +56,14 @@ def create_dir(dir_path):
5456

5557
def setup():
5658
if os.path.exists(temp_folder):
57-
_rmtree()
59+
_rmtree(temp_folder)
5860

5961
os.makedirs(temp_folder)
6062

6163

6264
def cleanup():
6365
if os.path.exists(temp_folder):
64-
_rmtree()
66+
_rmtree(temp_folder)
6567

6668
os_utils.reset_os()
6769

@@ -74,7 +76,7 @@ def cleanup():
7476
_original_env.clear()
7577

7678

77-
def _rmtree():
79+
def _rmtree(folder):
7880
exception = None
7981

8082
def on_rm_error(func, path, exc_info):
@@ -87,7 +89,7 @@ def on_rm_error(func, path, exc_info):
8789
if exception is None:
8890
exception = e
8991

90-
shutil.rmtree(temp_folder, onerror=on_rm_error)
92+
shutil.rmtree(folder, onerror=on_rm_error)
9193
if exception:
9294
raise exception
9395

@@ -134,7 +136,6 @@ def create_script_param_config(
134136
file_recursive=None,
135137
file_type=None,
136138
file_extensions=None):
137-
138139
conf = {'name': param_name}
139140

140141
if type is not None:
@@ -308,6 +309,42 @@ def set_env_value(key, value):
308309
os.environ[key] = value
309310

310311

312+
def assert_large_dict_equal(expected, actual, testcase):
313+
if len(expected) < 20 and len(actual) < 20:
314+
testcase.assertEqual(expected, actual)
315+
return
316+
317+
if expected == actual:
318+
return
319+
320+
diff_expected = {}
321+
diff_actual = {}
322+
too_large_diff = False
323+
324+
all_keys = set()
325+
all_keys.update(expected.keys())
326+
all_keys.update(actual.keys())
327+
for key in all_keys:
328+
expected_value = expected.get(key)
329+
actual_value = actual.get(key)
330+
331+
if expected_value == actual_value:
332+
continue
333+
334+
diff_expected[key] = expected_value
335+
diff_actual[key] = actual_value
336+
337+
if len(diff_expected) >= 50:
338+
too_large_diff = True
339+
break
340+
341+
message = 'Showing only different elements'
342+
if too_large_diff:
343+
message += ' (limited to 50)'
344+
345+
testcase.assertEqual(diff_expected, diff_actual, message)
346+
347+
311348
class _MockProcessWrapper(ProcessWrapper):
312349
def __init__(self, executor, command, working_directory):
313350
super().__init__(command, working_directory)
@@ -374,4 +411,3 @@ def is_allowed(self, user_id, allowed_users):
374411

375412
def is_admin(self, user_id):
376413
return True
377-

0 commit comments

Comments
 (0)