From a8016a480f3c5e13a7905f7d46fc2d3d47bb986d Mon Sep 17 00:00:00 2001 From: Sourcery AI <> Date: Tue, 28 Feb 2023 19:07:46 +0000 Subject: [PATCH] 'Refactored by Sourcery' --- diskcache/core.py | 172 ++++++++++++--------------------- diskcache/djangocache.py | 2 +- diskcache/persistent.py | 15 ++- diskcache/recipes.py | 4 +- tests/benchmark_core.py | 40 ++++---- tests/benchmark_djangocache.py | 6 +- tests/benchmark_glob.py | 6 +- tests/benchmark_incr.py | 4 +- tests/plot.py | 17 ++-- tests/stress_test_core.py | 27 +++--- tests/stress_test_deque_mp.py | 11 ++- tests/stress_test_fanout.py | 19 ++-- tests/stress_test_index_mp.py | 8 +- tests/test_core.py | 22 +++-- tests/test_deque.py | 4 +- tests/test_djangocache.py | 16 ++- tests/test_fanout.py | 4 +- tests/utils.py | 44 +++++---- 18 files changed, 176 insertions(+), 245 deletions(-) diff --git a/diskcache/core.py b/diskcache/core.py index 0c8fd2c..10a8cba 100644 --- a/diskcache/core.py +++ b/diskcache/core.py @@ -51,7 +51,7 @@ def full_name(func): name = func.__qualname__ except AttributeError: name = func.__name__ - return func.__module__ + '.' + name + return f'{func.__module__}.{name}' ############################################################################ # END Python 2/3 Shims @@ -69,7 +69,7 @@ def __new__(cls, name): return tuple.__new__(cls, (name,)) def __repr__(self): - return '%s' % self[0] + return f'{self[0]}' DBNAME = 'cache.db' ENOVAL = Constant('ENOVAL') @@ -233,13 +233,12 @@ def store(self, value, read, key=UNKNOWN): elif type_value is BytesType: if len(value) < min_file_size: return 0, MODE_RAW, None, sqlite3.Binary(value) - else: - filename, full_path = self.filename(key, value) + filename, full_path = self.filename(key, value) - with open(full_path, 'wb') as writer: - writer.write(value) + with open(full_path, 'wb') as writer: + writer.write(value) - return len(value), MODE_BINARY, filename, None + return len(value), MODE_BINARY, filename, None elif type_value is TextType: filename, full_path = self.filename(key, value) @@ -264,13 +263,12 @@ def store(self, value, read, key=UNKNOWN): if len(result) < min_file_size: return 0, MODE_PICKLE, None, sqlite3.Binary(result) - else: - filename, full_path = self.filename(key, value) + filename, full_path = self.filename(key, value) - with open(full_path, 'wb') as writer: - writer.write(result) + with open(full_path, 'wb') as writer: + writer.write(result) - return len(result), MODE_PICKLE, filename, None + return len(result), MODE_PICKLE, filename, None def fetch(self, mode, filename, value, read): @@ -290,19 +288,17 @@ def fetch(self, mode, filename, value, read): elif mode == MODE_BINARY: if read: return open(op.join(self._directory, filename), 'rb') - else: - with open(op.join(self._directory, filename), 'rb') as reader: - return reader.read() + with open(op.join(self._directory, filename), 'rb') as reader: + return reader.read() elif mode == MODE_TEXT: full_path = op.join(self._directory, filename) with io_open(full_path, 'r', encoding='UTF-8') as reader: return reader.read() elif mode == MODE_PICKLE: - if value is None: - with open(op.join(self._directory, filename), 'rb') as reader: - return pickle.load(reader) - else: + if value is not None: return pickle.load(BytesIO(value)) + with open(op.join(self._directory, filename), 'rb') as reader: + return pickle.load(reader) def filename(self, key=UNKNOWN, value=UNKNOWN): @@ -326,7 +322,7 @@ def filename(self, key=UNKNOWN, value=UNKNOWN): # pylint: disable=unused-argument hex_name = codecs.encode(os.urandom(16), 'hex').decode('utf-8') sub_dir = op.join(hex_name[:2], hex_name[2:4]) - name = hex_name[4:] + '.val' + name = f'{hex_name[4:]}.val' directory = op.join(self._directory, sub_dir) try: @@ -468,8 +464,8 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): directory = op.expanduser(directory) directory = op.expandvars(directory) - self._directory = directory self._timeout = 0 # Manually handle retries during initialization. + self._directory = directory self._local = threading.local() self._txn_id = None @@ -480,8 +476,7 @@ def __init__(self, directory=None, timeout=60, disk=Disk, **settings): if error.errno != errno.EEXIST: raise EnvironmentError( error.errno, - 'Cache directory "%s" does not exist' - ' and could not be created' % self._directory + f'Cache directory "{self._directory}" does not exist and could not be created', ) sql = self._sql_retry @@ -824,13 +819,10 @@ def set(self, key, value, expire=None, read=False, tag=None, retry=False): # need cleanup. with self._transact(retry, filename) as (sql, cleanup): - rows = sql( - 'SELECT rowid, filename FROM Cache' - ' WHERE key = ? AND raw = ?', + if rows := sql( + 'SELECT rowid, filename FROM Cache' ' WHERE key = ? AND raw = ?', (db_key, raw), - ).fetchall() - - if rows: + ).fetchall(): (rowid, old_filename), = rows cleanup(old_filename) self._row_update(rowid, now, columns) @@ -919,13 +911,8 @@ def _cull(self, now, sql, cleanup, limit=None): ) select_expired = select_expired_template % 'filename' - rows = sql(select_expired, (now, cull_limit)).fetchall() - - if rows: - delete_expired = ( - 'DELETE FROM Cache WHERE rowid IN (%s)' - % (select_expired_template % 'rowid') - ) + if rows := sql(select_expired, (now, cull_limit)).fetchall(): + delete_expired = f"DELETE FROM Cache WHERE rowid IN ({select_expired_template % 'rowid'})" sql(delete_expired, (now, cull_limit)) for filename, in rows: @@ -944,9 +931,7 @@ def _cull(self, now, sql, cleanup, limit=None): return select_filename = select_policy.format(fields='filename', now=now) - rows = sql(select_filename, (cull_limit,)).fetchall() - - if rows: + if rows := sql(select_filename, (cull_limit,)).fetchall(): delete = ( 'DELETE FROM Cache WHERE rowid IN (%s)' % (select_policy.format(fields='rowid', now=now)) @@ -976,13 +961,11 @@ def touch(self, key, expire=None, retry=False): expire_time = None if expire is None else now + expire with self._transact(retry) as (sql, _): - rows = sql( + if rows := sql( 'SELECT rowid, expire_time FROM Cache' ' WHERE key = ? AND raw = ?', (db_key, raw), - ).fetchall() - - if rows: + ).fetchall(): (rowid, old_expire_time), = rows if old_expire_time is None or old_expire_time > now: @@ -1026,13 +1009,11 @@ def add(self, key, value, expire=None, read=False, tag=None, retry=False): columns = (expire_time, tag, size, mode, filename, db_value) with self._transact(retry, filename) as (sql, cleanup): - rows = sql( + if rows := sql( 'SELECT rowid, filename, expire_time FROM Cache' ' WHERE key = ? AND raw = ?', (db_key, raw), - ).fetchall() - - if rows: + ).fetchall(): (rowid, old_filename, old_expire_time), = rows if old_expire_time is None or old_expire_time > now: @@ -1113,9 +1094,9 @@ def incr(self, key, delta=1, default=0, retry=False): update_column = EVICTION_POLICY[self.eviction_policy]['get'] if update_column is not None: - columns += ', ' + update_column.format(now=now) + columns += f', {update_column.format(now=now)}' - update = 'UPDATE Cache SET %s WHERE rowid = ?' % columns + update = f'UPDATE Cache SET {columns} WHERE rowid = ?' sql(update, (now, value, rowid)) return value @@ -1222,21 +1203,20 @@ def get(self, key, default=None, read=False, expire_time=False, tag=False, try: value = self._disk.fetch(mode, filename, db_value, read) except IOError as error: - if error.errno == errno.ENOENT: - # Key was deleted before we could retrieve result. - if self.statistics: - sql(cache_miss) - return default - else: + if error.errno != errno.ENOENT: raise + # Key was deleted before we could retrieve result. + if self.statistics: + sql(cache_miss) + return default if self.statistics: sql(cache_hit) now = time.time() - update = 'UPDATE Cache SET %s WHERE rowid = ?' - if update_column is not None: + update = 'UPDATE Cache SET %s WHERE rowid = ?' + sql(update % update_column.format(now=now), (rowid,)) if expire_time and tag: @@ -1466,32 +1446,22 @@ def push(self, value, prefix=None, side='back', expire=None, read=False, min_key = 0 max_key = 999999999999999 else: - min_key = prefix + '-000000000000000' - max_key = prefix + '-999999999999999' + min_key = f'{prefix}-000000000000000' + max_key = f'{prefix}-999999999999999' now = time.time() - raw = True expire_time = None if expire is None else now + expire size, mode, filename, db_value = self._disk.store(value, read) columns = (expire_time, tag, size, mode, filename, db_value) order = {'back': 'DESC', 'front': 'ASC'} - select = ( - 'SELECT key FROM Cache' - ' WHERE ? < key AND key < ? AND raw = ?' - ' ORDER BY key %s LIMIT 1' - ) % order[side] + select = f'SELECT key FROM Cache WHERE ? < key AND key < ? AND raw = ? ORDER BY key {order[side]} LIMIT 1' + raw = True with self._transact(retry, filename) as (sql, cleanup): - rows = sql(select, (min_key, max_key, raw)).fetchall() - - if rows: + if rows := sql(select, (min_key, max_key, raw)).fetchall(): (key,), = rows - if prefix is not None: - num = int(key[(key.rfind('-') + 1):]) - else: - num = key - + num = int(key[(key.rfind('-') + 1):]) if prefix is not None else key if side == 'back': num += 1 else: @@ -1500,11 +1470,7 @@ def push(self, value, prefix=None, side='back', expire=None, read=False, else: num = 500000000000000 - if prefix is not None: - db_key = '{0}-{1:015d}'.format(prefix, num) - else: - db_key = num - + db_key = '{0}-{1:015d}'.format(prefix, num) if prefix is not None else num self._row_insert(db_key, raw, now, columns) self._cull(now, sql, cleanup) @@ -1570,8 +1536,8 @@ def pull(self, prefix=None, default=(None, None), side='front', min_key = 0 max_key = 999999999999999 else: - min_key = prefix + '-000000000000000' - max_key = prefix + '-999999999999999' + min_key = f'{prefix}-000000000000000' + max_key = f'{prefix}-999999999999999' order = {'front': 'ASC', 'back': 'DESC'} select = ( @@ -1681,8 +1647,8 @@ def peek(self, prefix=None, default=(None, None), side='front', min_key = 0 max_key = 999999999999999 else: - min_key = prefix + '-000000000000000' - max_key = prefix + '-999999999999999' + min_key = f'{prefix}-000000000000000' + max_key = f'{prefix}-999999999999999' order = {'front': 'ASC', 'back': 'DESC'} select = ( @@ -1707,12 +1673,11 @@ def peek(self, prefix=None, default=(None, None), side='front', (rowid, key, db_expire, db_tag, mode, name, db_value), = rows - if db_expire is not None and db_expire < time.time(): - sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) - cleanup(name) - else: + if db_expire is None or db_expire >= time.time(): break + sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) + cleanup(name) try: value = self._disk.fetch(mode, name, db_value, False) except IOError as error: @@ -1968,7 +1933,7 @@ def check(self, fix=False, retry=False): continue - warnings.warn('file not found: %s' % full_path) + warnings.warn(f'file not found: {full_path}') if fix: sql('DELETE FROM Cache WHERE rowid = ?', (rowid,)) @@ -1983,7 +1948,7 @@ def check(self, fix=False, retry=False): if DBNAME in full_path: continue - message = 'unknown file: %s' % full_path + message = f'unknown file: {full_path}' warnings.warn(message, UnknownFileWarning) if fix: @@ -1993,7 +1958,7 @@ def check(self, fix=False, retry=False): for dirpath, dirs, files in os.walk(self._directory): if not (dirs or files): - message = 'empty directory: %s' % dirpath + message = f'empty directory: {dirpath}' warnings.warn(message, EmptyDirWarning) if fix: @@ -2260,9 +2225,7 @@ def iterkeys(self, reverse=False): ' ORDER BY key ASC, raw ASC LIMIT ?' ) - row = sql(select).fetchall() - - if row: + if row := sql(select).fetchall(): (key, raw), = row else: return @@ -2288,22 +2251,14 @@ def _iter(self, ascending=True): if max_rowid is None: return - bound = max_rowid + 1 limit = 100 _disk_get = self._disk.get + bound = max_rowid + 1 rowid = 0 if ascending else bound - select = ( - 'SELECT rowid, key, raw FROM Cache' - ' WHERE ? < rowid AND rowid < ?' - ' ORDER BY rowid %s LIMIT ?' - ) % ('ASC' if ascending else 'DESC') + select = f"SELECT rowid, key, raw FROM Cache WHERE ? < rowid AND rowid < ? ORDER BY rowid {'ASC' if ascending else 'DESC'} LIMIT ?" while True: - if ascending: - args = (rowid, bound, limit) - else: - args = (0, rowid, limit) - + args = (rowid, bound, limit) if ascending else (0, rowid, limit) rows = sql(select, args).fetchall() if not rows: @@ -2354,8 +2309,7 @@ def volume(self): """ (page_count,), = self._sql('PRAGMA page_count').fetchall() - total_size = self._page_size * page_count + self.reset('size') - return total_size + return self._page_size * page_count + self.reset('size') def close(self): @@ -2369,10 +2323,8 @@ def close(self): con.close() - try: + with cl.suppress(AttributeError): delattr(self._local, 'con') - except AttributeError: - pass def __enter__(self): @@ -2459,12 +2411,12 @@ def reset(self, key, value=ENOVAL, update=True): while True: try: try: - (old_value,), = sql('PRAGMA %s' % (pragma)).fetchall() + (old_value,), = sql(f'PRAGMA {pragma}').fetchall() update = old_value != value except ValueError: update = True if update: - sql('PRAGMA %s = %s' % (pragma, value)).fetchall() + sql(f'PRAGMA {pragma} = {value}').fetchall() break except sqlite3.OperationalError as exc: if str(exc) != 'database is locked': diff --git a/diskcache/djangocache.py b/diskcache/djangocache.py index 997b852..9a640a9 100644 --- a/diskcache/djangocache.py +++ b/diskcache/djangocache.py @@ -228,7 +228,7 @@ def incr(self, key, delta=1, version=None, default=None, retry=True): try: return self._cache.incr(key, delta, default, retry) except KeyError: - raise ValueError("Key '%s' not found" % key) + raise ValueError(f"Key '{key}' not found") def decr(self, key, delta=1, version=None, default=None, retry=True): diff --git a/diskcache/persistent.py b/diskcache/persistent.py index 9de5835..12b8209 100644 --- a/diskcache/persistent.py +++ b/diskcache/persistent.py @@ -386,7 +386,7 @@ def count(self, value): :return: count of items equal to value in deque """ - return sum(1 for item in self if value == item) + return sum(value == item for item in self) def extend(self, iterable): @@ -600,7 +600,7 @@ def rotate(self, steps=1): """ if not isinstance(steps, int): type_name = type(steps).__name__ - raise TypeError('integer argument expected, got %s' % type_name) + raise TypeError(f'integer argument expected, got {type_name}') len_self = len(self) @@ -1275,13 +1275,12 @@ def __eq__(self, other): if len(self) != len(other): return False - if isinstance(other, (Index, OrderedDict)): - alpha = ((key, self[key]) for key in self) - beta = ((key, other[key]) for key in other) - pairs = zip(alpha, beta) - return not any(a != x or b != y for (a, b), (x, y) in pairs) - else: + if not isinstance(other, (Index, OrderedDict)): return all(self[key] == other.get(key, ENOVAL) for key in self) + alpha = ((key, self[key]) for key in self) + beta = ((key, other[key]) for key in other) + pairs = zip(alpha, beta) + return not any(a != x or b != y for (a, b), (x, y) in pairs) def __ne__(self, other): diff --git a/diskcache/recipes.py b/diskcache/recipes.py index fb64250..55972a7 100644 --- a/diskcache/recipes.py +++ b/diskcache/recipes.py @@ -140,7 +140,7 @@ def acquire(self): "Acquire lock by incrementing count using spin-lock algorithm." pid = os.getpid() tid = get_ident() - pid_tid = '{}-{}'.format(pid, tid) + pid_tid = f'{pid}-{tid}' while True: with self._cache.transact(retry=True): @@ -157,7 +157,7 @@ def release(self): "Release lock by decrementing count." pid = os.getpid() tid = get_ident() - pid_tid = '{}-{}'.format(pid, tid) + pid_tid = f'{pid}-{tid}' with self._cache.transact(retry=True): value, count = self._cache.get(self._key, default=(None, 0)) diff --git a/tests/benchmark_core.py b/tests/benchmark_core.py index 1811de0..2ceee3b 100644 --- a/tests/benchmark_core.py +++ b/tests/benchmark_core.py @@ -6,6 +6,7 @@ """ + from __future__ import print_function import collections as co @@ -30,30 +31,27 @@ RANGE = 100 WARMUP = int(1e3) -caches = [] - - ############################################################################### # Disk Cache Benchmarks ############################################################################### import diskcache -caches.append(('diskcache.Cache', diskcache.Cache, ('tmp',), {},)) -caches.append(( - 'diskcache.FanoutCache(shards=4, timeout=1.0)', - diskcache.FanoutCache, - ('tmp',), - {'shards': 4, 'timeout': 1.0} -)) -caches.append(( - 'diskcache.FanoutCache(shards=8, timeout=0.010)', - diskcache.FanoutCache, - ('tmp',), - {'shards': 8, 'timeout': 0.010} -)) - - +caches = [ + ('diskcache.Cache', diskcache.Cache, ('tmp',), {}), + ( + 'diskcache.FanoutCache(shards=4, timeout=1.0)', + diskcache.FanoutCache, + ('tmp',), + {'shards': 4, 'timeout': 1.0}, + ), + ( + 'diskcache.FanoutCache(shards=8, timeout=0.010)', + diskcache.FanoutCache, + ('tmp',), + {'shards': 8, 'timeout': 0.010}, + ), +] ############################################################################### # PyLibMC Benchmarks ############################################################################### @@ -102,20 +100,18 @@ def worker(num, kind, args, kwargs): value = str(count).encode('utf-8') * random.randrange(1, 100) choice = random.random() + start = time.time() if choice < 0.900: - start = time.time() result = obj.get(key) end = time.time() miss = result is None action = 'get' elif choice < 0.990: - start = time.time() result = obj.set(key, value) end = time.time() miss = result == False action = 'set' else: - start = time.time() result = obj.delete(key) end = time.time() miss = result == False @@ -125,7 +121,7 @@ def worker(num, kind, args, kwargs): delta = end - start timings[action].append(delta) if miss: - timings[action + '-miss'].append(delta) + timings[f'{action}-miss'].append(delta) with open('output-%d.pkl' % num, 'wb') as writer: pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/tests/benchmark_djangocache.py b/tests/benchmark_djangocache.py index 898188f..58b0b9f 100644 --- a/tests/benchmark_djangocache.py +++ b/tests/benchmark_djangocache.py @@ -55,20 +55,18 @@ def worker(num, name): value = str(count).encode('utf-8') * random.randrange(1, 100) choice = random.random() + start = time.time() if choice < 0.900: - start = time.time() result = obj.get(key) end = time.time() miss = result is None action = 'get' elif choice < 0.990: - start = time.time() result = obj.set(key, value) end = time.time() miss = result == False action = 'set' else: - start = time.time() result = obj.delete(key) end = time.time() miss = result == False @@ -78,7 +76,7 @@ def worker(num, name): delta = end - start timings[action].append(delta) if miss: - timings[action + '-miss'].append(delta) + timings[f'{action}-miss'].append(delta) with open('output-%d.pkl' % num, 'wb') as writer: pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL) diff --git a/tests/benchmark_glob.py b/tests/benchmark_glob.py index 0402ef8..c76a620 100644 --- a/tests/benchmark_glob.py +++ b/tests/benchmark_glob.py @@ -15,7 +15,7 @@ size = 12 cols = ('Count', 'Time') -template = ' '.join(['%' + str(size) + 's'] * len(cols)) +template = ' '.join([f'%{size}s'] * len(cols)) print() print(' '.join(['=' * size] * len(cols))) @@ -26,9 +26,9 @@ for count in [10 ** exp for exp in range(6)]: for value in range(count): - with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer: + with open(op.join('tmp', f'{value}.tmp'), 'wb') as writer: pass - + delta = timeit.timeit( stmt="glob.glob1('tmp', '*.tmp')", setup='import glob', diff --git a/tests/benchmark_incr.py b/tests/benchmark_incr.py index 4a01628..7fa7cb7 100644 --- a/tests/benchmark_incr.py +++ b/tests/benchmark_incr.py @@ -30,7 +30,7 @@ def worker(num): end = time.time() values.append(end - start) - with open('output-%s.json' % num, 'w') as writer: + with open(f'output-{num}.json', 'w') as writer: json.dump(values, writer) @@ -53,7 +53,7 @@ def main(): for num in range(PROCS): values = [] - with open('output-%s.json' % num) as reader: + with open(f'output-{num}.json') as reader: values += json.load(reader) values.sort() diff --git a/tests/plot.py b/tests/plot.py index d8d1be0..0ecd61f 100644 --- a/tests/plot.py +++ b/tests/plot.py @@ -46,7 +46,7 @@ def parse_data(infile): if blocks.match(line): try: - name = title.match(lines[index + 1]).group(1) + name = title.match(lines[index + 1])[1] except: index += 1 continue @@ -90,18 +90,17 @@ def make_plot(data, action, save=False, show=False, limit=0.005): ticks = ('Median', 'P90', 'P99') index = (0, 1, 2) names = list(data) - bars = [] - - for pos, (name, color) in enumerate(zip(names, colors)): - bars.append(ax.bar( + bars = [ + ax.bar( [val + pos * width for val in index], [parse_timing(data[name][action][tick], limit) for tick in ticks], width, color=color, - )) - + ) + for pos, (name, color) in enumerate(zip(names, colors)) + ] ax.set_ylabel('Time (microseconds)') - ax.set_title('"%s" Time vs Percentile' % action) + ax.set_title(f'"{action}" Time vs Percentile') ax.set_xticks([val + width * (len(data) / 2) for val in index]) ax.set_xticklabels(ticks) @@ -118,7 +117,7 @@ def make_plot(data, action, save=False, show=False, limit=0.005): plt.show() if save: - plt.savefig('%s-%s.png' % (save, action), dpi=120, bbox_inches='tight') + plt.savefig(f'{save}-{action}.png', dpi=120, bbox_inches='tight') plt.close() diff --git a/tests/stress_test_core.py b/tests/stress_test_core.py index bce0d16..5a4bf0b 100644 --- a/tests/stress_test_core.py +++ b/tests/stress_test_core.py @@ -131,10 +131,10 @@ def worker(queue, eviction_policy, processes, threads): start = time.time() try: - if action == 'set': - cache.set(key, value, expire=EXPIRE) - elif action == 'get': + if action == 'get': result = cache.get(key) + elif action == 'set': + cache.set(key, value, expire=EXPIRE) else: assert action == 'delete' cache.delete(key) @@ -152,7 +152,7 @@ def worker(queue, eviction_policy, processes, threads): delta = stop - start timings[action].append(delta) if miss: - timings[action + '-miss'].append(delta) + timings[f'{action}-miss'].append(delta) queue.put(timings) @@ -160,7 +160,7 @@ def worker(queue, eviction_policy, processes, threads): def dispatch(num, eviction_policy, processes, threads): - with open('input-%s.pkl' % num, 'rb') as reader: + with open(f'input-{num}.pkl', 'rb') as reader: process_queue = pickle.load(reader) thread_queues = [Queue.Queue() for _ in range(threads)] @@ -194,7 +194,7 @@ def dispatch(num, eviction_policy, processes, threads): for key in data: timings[key].extend(data[key]) - with open('output-%s.pkl' % num, 'wb') as writer: + with open(f'output-{num}.pkl', 'wb') as writer: pickle.dump(timings, writer, protocol=2) @@ -217,12 +217,7 @@ def stress_test(create=True, delete=True, processes=1, threads=1): shutil.rmtree('tmp', ignore_errors=True) - if processes == 1: - # Use threads. - func = threading.Thread - else: - func = mp.Process - + func = threading.Thread if processes == 1 else mp.Process subprocs = [ func(target=dispatch, args=(num, eviction_policy, processes, threads)) for num in range(processes) @@ -236,7 +231,7 @@ def stress_test(create=True, delete=True, process_queue[index % processes].append(ops) for num in range(processes): - with open('input-%s.pkl' % num, 'wb') as writer: + with open(f'input-{num}.pkl', 'wb') as writer: pickle.dump(process_queue[num], writer, protocol=2) for process in subprocs: @@ -254,15 +249,15 @@ def stress_test(create=True, delete=True, timings = co.defaultdict(list) for num in range(processes): - with open('output-%s.pkl' % num, 'rb') as reader: + with open(f'output-{num}.pkl', 'rb') as reader: data = pickle.load(reader) for key in data: timings[key] += data[key] if delete: for num in range(processes): - os.remove('input-%s.pkl' % num) - os.remove('output-%s.pkl' % num) + os.remove(f'input-{num}.pkl') + os.remove(f'output-{num}.pkl') display(eviction_policy, timings) diff --git a/tests/stress_test_deque_mp.py b/tests/stress_test_deque_mp.py index 4624d71..4dd4a0c 100644 --- a/tests/stress_test_deque_mp.py +++ b/tests/stress_test_deque_mp.py @@ -101,11 +101,12 @@ def stress_rotate(deque): def stress(seed, deque): random.seed(seed) - for count in range(OPERATIONS): - if len(deque) > 100: - function = random.choice([stress_pop, stress_popleft]) - else: - function = random.choice(functions) + for _ in range(OPERATIONS): + function = ( + random.choice([stress_pop, stress_popleft]) + if len(deque) > 100 + else random.choice(functions) + ) function(deque) diff --git a/tests/stress_test_fanout.py b/tests/stress_test_fanout.py index 080b8d8..de7005e 100644 --- a/tests/stress_test_fanout.py +++ b/tests/stress_test_fanout.py @@ -152,7 +152,7 @@ def worker(queue, eviction_policy, processes, threads): def dispatch(num, eviction_policy, processes, threads): - with open('input-%s.pkl' % num, 'rb') as reader: + with open(f'input-{num}.pkl', 'rb') as reader: process_queue = pickle.load(reader) thread_queues = [Queue.Queue() for _ in range(threads)] @@ -186,7 +186,7 @@ def dispatch(num, eviction_policy, processes, threads): for key in data: timings[key].extend(data[key]) - with open('output-%s.pkl' % num, 'wb') as writer: + with open(f'output-{num}.pkl', 'wb') as writer: pickle.dump(timings, writer, protocol=2) @@ -209,12 +209,7 @@ def stress_test(create=True, delete=True, processes=1, threads=1): shutil.rmtree('tmp', ignore_errors=True) - if processes == 1: - # Use threads. - func = threading.Thread - else: - func = mp.Process - + func = threading.Thread if processes == 1 else mp.Process subprocs = [ func(target=dispatch, args=(num, eviction_policy, processes, threads)) for num in range(processes) @@ -228,7 +223,7 @@ def stress_test(create=True, delete=True, process_queue[index % processes].append(ops) for num in range(processes): - with open('input-%s.pkl' % num, 'wb') as writer: + with open(f'input-{num}.pkl', 'wb') as writer: pickle.dump(process_queue[num], writer, protocol=2) for process in subprocs: @@ -246,15 +241,15 @@ def stress_test(create=True, delete=True, timings = {'get': [], 'set': [], 'delete': [], 'self': 0.0} for num in range(processes): - with open('output-%s.pkl' % num, 'rb') as reader: + with open(f'output-{num}.pkl', 'rb') as reader: data = pickle.load(reader) for key in data: timings[key] += data[key] if delete: for num in range(processes): - os.remove('input-%s.pkl' % num) - os.remove('output-%s.pkl' % num) + os.remove(f'input-{num}.pkl') + os.remove(f'output-{num}.pkl') display(eviction_policy, timings) diff --git a/tests/stress_test_index_mp.py b/tests/stress_test_index_mp.py index b3ed813..30f10aa 100644 --- a/tests/stress_test_index_mp.py +++ b/tests/stress_test_index_mp.py @@ -71,17 +71,11 @@ def stress_popitem(index): def stress_iter(index): iterator = it.islice(index, 5) - for key in iterator: - pass - @register def stress_reversed(index): iterator = it.islice(reversed(index), 5) - for key in iterator: - pass - @register def stress_len(index): @@ -90,7 +84,7 @@ def stress_len(index): def stress(seed, index): random.seed(seed) - for count in range(OPERATIONS): + for _ in range(OPERATIONS): function = random.choice(functions) function(index) diff --git a/tests/test_core.py b/tests/test_core.py index 7e79995..90a3d1d 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -167,15 +167,19 @@ def test_pragma_error(cache): def test_close_error(cache): + + + class LocalTest(object): def __init__(self): self._calls = 0 + def __getattr__(self, name): if self._calls: raise AttributeError - else: - self._calls += 1 - return mock.Mock() + self._calls += 1 + return mock.Mock() + with mock.patch.object(cache, '_local', LocalTest()): cache.close() @@ -219,7 +223,7 @@ def test_getsetdel(cache): for value, (key, _) in enumerate(values): assert cache[key] == value - for _, (key, _) in enumerate(values): + for key, _ in values: del cache[key] assert len(cache) == 0 @@ -619,9 +623,7 @@ def test_remove_error(cache): except OSError: pass else: - if os.name == 'nt': - pass # File delete errors ignored on Windows. - else: + if os.name != 'nt': raise Exception('test_remove_error failed') @@ -636,7 +638,7 @@ def test_check(cache): with cache.get(0, read=True) as reader: full_path = reader.name - os.rename(full_path, full_path + '_moved') + os.rename(full_path, f'{full_path}_moved') with cache.get(1, read=True) as reader: full_path = reader.name @@ -1150,7 +1152,7 @@ def test_peekitem_ioerror_eacces(cache): def test_iterkeys(cache): - assert list(cache.iterkeys()) == [] + assert not list(cache.iterkeys()) def test_pickle(cache): @@ -1176,7 +1178,7 @@ def compare_pragmas(): pragma = key[7:] - result = cache._sql('PRAGMA %s' % pragma).fetchall() + result = cache._sql(f'PRAGMA {pragma}').fetchall() if result == [(value,)]: continue diff --git a/tests/test_deque.py b/tests/test_deque.py index 640cee2..9c3c543 100644 --- a/tests/test_deque.py +++ b/tests/test_deque.py @@ -136,8 +136,8 @@ def test_state(deque): def test_compare(deque): - assert not (deque == {}) - assert not (deque == [0]) + assert deque != {} + assert deque != [0] assert deque != [1] deque.append(0) assert deque <= [0] diff --git a/tests/test_djangocache.py b/tests/test_djangocache.py index f70b87c..88cd217 100644 --- a/tests/test_djangocache.py +++ b/tests/test_djangocache.py @@ -82,7 +82,7 @@ def f(): class C: - def m(n): + def m(self): return 24 @@ -108,7 +108,7 @@ def custom_key_func2(key, key_prefix, version): _caches_setting_base = { 'default': {}, - 'prefix': {'KEY_PREFIX': 'cacheprefix{}'.format(os.getpid())}, + 'prefix': {'KEY_PREFIX': f'cacheprefix{os.getpid()}'}, 'v2': {'VERSION': 2}, 'custom_key': {'KEY_FUNCTION': custom_key_func}, 'custom_key2': {'KEY_FUNCTION': custom_key_func2}, @@ -336,7 +336,7 @@ def test_unicode(self): self.assertEqual(cache.get(key), value) # Test `set_many` - for (key, value) in stuff.items(): + for key in stuff: cache.delete(key) cache.set_many(stuff) for (key, value) in stuff.items(): @@ -463,11 +463,9 @@ def _perform_cull_test(self, cull_cache, initial_count, final_count): # causing a cull. for i in range(1, initial_count): cull_cache.set('cull%d' % i, 'value', 1000) - count = 0 - # Count how many keys are left in the cache. - for i in range(1, initial_count): - if cull_cache.has_key('cull%d' % i): - count += 1 + count = sum( + 1 for i in range(1, initial_count) if cull_cache.has_key('cull%d' % i) + ) self.assertEqual(count, final_count) def test_cull(self): @@ -849,7 +847,7 @@ def test_get_or_set_version(self): self.assertIsNone(cache.get('brian', version=3)) def test_get_or_set_racing(self): - with mock.patch('%s.%s' % (settings.CACHES['default']['BACKEND'], 'add')) as cache_add: + with mock.patch(f"{settings.CACHES['default']['BACKEND']}.add") as cache_add: # Simulate cache.add() failing to add a value. In that case, the # default value should be returned. cache_add.return_value = False diff --git a/tests/test_fanout.py b/tests/test_fanout.py index a62f8a2..c875563 100644 --- a/tests/test_fanout.py +++ b/tests/test_fanout.py @@ -277,7 +277,7 @@ def test_getsetdel(cache): for value, (key, _) in enumerate(values): assert cache[key] == value - for _, (key, _) in enumerate(values): + for key, _ in values: del cache[key] assert len(cache) == 0 @@ -506,7 +506,7 @@ def test_iter_expire(cache): time.sleep(0.1) assert set(cache) == set(range(100)) cache.expire() - assert set(cache) == set() + assert not set(cache) def test_reversed(cache): diff --git a/tests/utils.py b/tests/utils.py index f2370da..2728bf8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -19,19 +19,16 @@ def percentile(sequence, percent): def secs(value): - units = ['s ', 'ms', 'us', 'ns'] pos = 0 - if value is None: - return ' 0.000ns' - elif value == 0: + if value is None or value == 0: return ' 0.000ns' - else: - for unit in units: - if value > 1: - return '%7.3f' % value + unit - else: - value *= 1000 + units = ['s ', 'ms', 'us', 'ns'] + for unit in units: + if value > 1: + return '%7.3f' % value + unit + else: + value *= 1000 def run(*args): @@ -86,7 +83,7 @@ def display(name, timings): print() print(' '.join(['=' * 9] * len(cols))) - print('Timings for %s' % name) + print(f'Timings for {name}') print('-'.join(['-' * 9] * len(cols))) print(template % cols) print(' '.join(['=' * 9] * len(cols))) @@ -98,16 +95,21 @@ def display(name, timings): len_total += len(values) sum_total += sum(values) - print(template % ( - action, - len(values), - len(timings.get(action + '-miss', [])), - secs(percentile(values, 0.5)), - secs(percentile(values, 0.9)), - secs(percentile(values, 0.99)), - secs(percentile(values, 1.0)), - secs(sum(values)), - )) + print( + ( + template + % ( + action, + len(values), + len(timings.get(f'{action}-miss', [])), + secs(percentile(values, 0.5)), + secs(percentile(values, 0.9)), + secs(percentile(values, 0.99)), + secs(percentile(values, 1.0)), + secs(sum(values)), + ) + ) + ) totals = ('Total', len_total, '', '', '', '', '', secs(sum_total)) print(template % totals)