Skip to content

Commit aceb473

Browse files
chillaqchillaq
committed
added sha check and integration tests
1 parent cfd1296 commit aceb473

6 files changed

Lines changed: 986 additions & 47 deletions

File tree

‎splitio/sync/segment.py‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ def __init__(self, segment_folder, split_storage, segment_storage):
213213
self._segment_folder = segment_folder
214214
self._split_storage = split_storage
215215
self._segment_storage = segment_storage
216+
self._segment_sha = {}
216217

217218
def synchronize_segment(self, segment_name, till=None):
218219
"""
@@ -230,17 +231,20 @@ def synchronize_segment(self, segment_name, till=None):
230231
try:
231232
fetched = self._read_segment_from_json_file(segment_name)
232233
if not self.segment_exist_in_storage(segment_name):
234+
self._segment_sha[segment_name] = self._get_sha(json.dumps(fetched))
233235
self._segment_storage.put(segments.from_raw(fetched))
234236
_LOGGER.debug("segment %s is added to storage", segment_name)
235237
else:
236-
if self._segment_storage.get_change_number(segment_name) <= fetched['till']:
237-
self._segment_storage.update(
238-
segment_name,
239-
fetched['added'],
240-
fetched['removed'],
241-
fetched['till']
242-
)
243-
_LOGGER.debug("segment %s is updated", segment_name)
238+
if self._get_sha(json.dumps(fetched)) != self._segment_sha[segment_name]:
239+
self._segment_sha[segment_name] = self._get_sha(json.dumps(fetched))
240+
if self._segment_storage.get_change_number(segment_name) <= fetched['till']:
241+
self._segment_storage.update(
242+
segment_name,
243+
fetched['added'],
244+
fetched['removed'],
245+
fetched['till']
246+
)
247+
_LOGGER.debug("segment %s is updated", segment_name)
244248
except Exception as e:
245249
_LOGGER.error("Could not fetch segment: %s \n" + str(e), segment_name)
246250
return False
@@ -269,6 +273,9 @@ def _sanitize_segment(self, segment):
269273
"""To be implemented."""
270274
return segment
271275

276+
def _get_sha(self, fetched):
277+
return hashlib.sha256(fetched.encode()).hexdigest()
278+
272279
def synchronize_segments(self, segment_names = None):
273280
"""
274281
Loop through given segment names and synchronize each one.

‎splitio/sync/split.py‎

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import yaml
66
import time
77
import json
8+
import hashlib
89
from enum import Enum
910

1011
from splitio.api import APIException
@@ -175,6 +176,7 @@ def __init__(self, filename, split_storage, localhost_mode=LocalhostMode.LEGACY)
175176
self._filename = filename
176177
self._split_storage = split_storage
177178
self._localhost_mode = localhost_mode
179+
self._current_json_sha = "-1"
178180

179181
@staticmethod
180182
def _make_split(split_name, conditions, configs=None):
@@ -351,24 +353,21 @@ def _synchronize_json(self):
351353
:return: segment names string array
352354
:rtype: [str]
353355
"""
354-
fetched, since, till = self._read_splits_from_json_file(self._filename)
356+
fetched, till = self._read_splits_from_json_file(self._filename)
355357
segment_list = set()
356-
if self._split_storage.get_change_number() <= till:
357-
to_delete = []
358-
if since == -1:
359-
to_delete = [name for name in self._split_storage.get_split_names()
360-
if name not in json.dumps(fetched)]
361-
for split in fetched:
362-
parsed = splits.from_raw(split)
363-
_LOGGER.debug("split %s is updated", parsed.name)
364-
self._split_storage.put(parsed)
365-
366-
segment_list.update(set(parsed.get_segment_names()))
367-
368-
for split in to_delete:
369-
self._split_storage.remove(split)
358+
if self._get_sha(json.dumps(fetched)) != self._current_json_sha:
359+
self._current_json_sha = self._get_sha(json.dumps(fetched))
360+
if self._split_storage.get_change_number() <= till:
361+
for split in fetched:
362+
if split['status'] == splits.Status.ACTIVE.value:
363+
parsed = splits.from_raw(split)
364+
self._split_storage.put(parsed)
365+
_LOGGER.debug("split %s is updated", parsed.name)
366+
segment_list.update(set(parsed.get_segment_names()))
367+
else:
368+
self._split_storage.remove(split['name'])
370369

371-
self._split_storage.set_change_number(till)
370+
self._split_storage.set_change_number(till)
372371

373372
return segment_list
374373

@@ -385,14 +384,17 @@ def _read_splits_from_json_file(self, filename):
385384
try:
386385
with open(filename, 'r') as flo:
387386
json_obj = json.load(flo)
388-
since = json_obj['since']
389-
till = json_obj['till']
387+
till = json_obj['till'] if 'till' in json_obj else -1
390388
parsed = json_obj['splits']
391389
santitized_split = self._sanitize_split(parsed)
392-
return santitized_split, since, till
390+
flo.close
391+
return santitized_split, till
393392
except IOError as exc:
394393
raise ValueError("Error parsing file %s. Make sure it's readable." % filename) from exc
395394

396395
def _sanitize_split(self, split):
397396
"""To be implemented."""
398397
return split
398+
399+
def _get_sha(self, fetched):
400+
return hashlib.sha256(fetched.encode()).hexdigest()

0 commit comments

Comments
 (0)