Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 62 additions & 110 deletions diskcache/core.py

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion diskcache/djangocache.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ def incr(self, key, delta=1, version=None, default=None, retry=True):
try:
return self._cache.incr(key, delta, default, retry)
except KeyError:
raise ValueError("Key '%s' not found" % key)
raise ValueError(f"Key '{key}' not found")


def decr(self, key, delta=1, version=None, default=None, retry=True):
Expand Down
15 changes: 7 additions & 8 deletions diskcache/persistent.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,7 @@ def count(self, value):
:return: count of items equal to value in deque

"""
return sum(1 for item in self if value == item)
return sum(value == item for item in self)


def extend(self, iterable):
Expand Down Expand Up @@ -600,7 +600,7 @@ def rotate(self, steps=1):
"""
if not isinstance(steps, int):
type_name = type(steps).__name__
raise TypeError('integer argument expected, got %s' % type_name)
raise TypeError(f'integer argument expected, got {type_name}')

len_self = len(self)

Expand Down Expand Up @@ -1275,13 +1275,12 @@ def __eq__(self, other):
if len(self) != len(other):
return False

if isinstance(other, (Index, OrderedDict)):
alpha = ((key, self[key]) for key in self)
beta = ((key, other[key]) for key in other)
pairs = zip(alpha, beta)
return not any(a != x or b != y for (a, b), (x, y) in pairs)
else:
if not isinstance(other, (Index, OrderedDict)):
return all(self[key] == other.get(key, ENOVAL) for key in self)
alpha = ((key, self[key]) for key in self)
beta = ((key, other[key]) for key in other)
pairs = zip(alpha, beta)
return not any(a != x or b != y for (a, b), (x, y) in pairs)


def __ne__(self, other):
Expand Down
4 changes: 2 additions & 2 deletions diskcache/recipes.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def acquire(self):
"Acquire lock by incrementing count using spin-lock algorithm."
pid = os.getpid()
tid = get_ident()
pid_tid = '{}-{}'.format(pid, tid)
pid_tid = f'{pid}-{tid}'

while True:
with self._cache.transact(retry=True):
Expand All @@ -157,7 +157,7 @@ def release(self):
"Release lock by decrementing count."
pid = os.getpid()
tid = get_ident()
pid_tid = '{}-{}'.format(pid, tid)
pid_tid = f'{pid}-{tid}'

with self._cache.transact(retry=True):
value, count = self._cache.get(self._key, default=(None, 0))
Expand Down
40 changes: 18 additions & 22 deletions tests/benchmark_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

"""


from __future__ import print_function

import collections as co
Expand All @@ -30,30 +31,27 @@
RANGE = 100
WARMUP = int(1e3)

caches = []


###############################################################################
# Disk Cache Benchmarks
###############################################################################

import diskcache

caches.append(('diskcache.Cache', diskcache.Cache, ('tmp',), {},))
caches.append((
'diskcache.FanoutCache(shards=4, timeout=1.0)',
diskcache.FanoutCache,
('tmp',),
{'shards': 4, 'timeout': 1.0}
))
caches.append((
'diskcache.FanoutCache(shards=8, timeout=0.010)',
diskcache.FanoutCache,
('tmp',),
{'shards': 8, 'timeout': 0.010}
))


caches = [
('diskcache.Cache', diskcache.Cache, ('tmp',), {}),
(
'diskcache.FanoutCache(shards=4, timeout=1.0)',
diskcache.FanoutCache,
('tmp',),
{'shards': 4, 'timeout': 1.0},
),
(
'diskcache.FanoutCache(shards=8, timeout=0.010)',
diskcache.FanoutCache,
('tmp',),
{'shards': 8, 'timeout': 0.010},
),
]
###############################################################################
# PyLibMC Benchmarks
###############################################################################
Expand Down Expand Up @@ -102,20 +100,18 @@ def worker(num, kind, args, kwargs):
value = str(count).encode('utf-8') * random.randrange(1, 100)
choice = random.random()

start = time.time()
if choice < 0.900:
start = time.time()
result = obj.get(key)
end = time.time()
miss = result is None
action = 'get'
elif choice < 0.990:
start = time.time()
result = obj.set(key, value)
end = time.time()
miss = result == False
action = 'set'
else:
start = time.time()
result = obj.delete(key)
end = time.time()
miss = result == False
Expand All @@ -125,7 +121,7 @@ def worker(num, kind, args, kwargs):
delta = end - start
timings[action].append(delta)
if miss:
timings[action + '-miss'].append(delta)
timings[f'{action}-miss'].append(delta)

with open('output-%d.pkl' % num, 'wb') as writer:
pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL)
Expand Down
6 changes: 2 additions & 4 deletions tests/benchmark_djangocache.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,20 +55,18 @@ def worker(num, name):
value = str(count).encode('utf-8') * random.randrange(1, 100)
choice = random.random()

start = time.time()
if choice < 0.900:
start = time.time()
result = obj.get(key)
end = time.time()
miss = result is None
action = 'get'
elif choice < 0.990:
start = time.time()
result = obj.set(key, value)
end = time.time()
miss = result == False
action = 'set'
else:
start = time.time()
result = obj.delete(key)
end = time.time()
miss = result == False
Expand All @@ -78,7 +76,7 @@ def worker(num, name):
delta = end - start
timings[action].append(delta)
if miss:
timings[action + '-miss'].append(delta)
timings[f'{action}-miss'].append(delta)

with open('output-%d.pkl' % num, 'wb') as writer:
pickle.dump(timings, writer, protocol=pickle.HIGHEST_PROTOCOL)
Expand Down
6 changes: 3 additions & 3 deletions tests/benchmark_glob.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

size = 12
cols = ('Count', 'Time')
template = ' '.join(['%' + str(size) + 's'] * len(cols))
template = ' '.join([f'%{size}s'] * len(cols))

print()
print(' '.join(['=' * size] * len(cols)))
Expand All @@ -26,9 +26,9 @@

for count in [10 ** exp for exp in range(6)]:
for value in range(count):
with open(op.join('tmp', '%s.tmp' % value), 'wb') as writer:
with open(op.join('tmp', f'{value}.tmp'), 'wb') as writer:
pass

delta = timeit.timeit(
stmt="glob.glob1('tmp', '*.tmp')",
setup='import glob',
Expand Down
4 changes: 2 additions & 2 deletions tests/benchmark_incr.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def worker(num):
end = time.time()
values.append(end - start)

with open('output-%s.json' % num, 'w') as writer:
with open(f'output-{num}.json', 'w') as writer:
json.dump(values, writer)


Expand All @@ -53,7 +53,7 @@ def main():

for num in range(PROCS):
values = []
with open('output-%s.json' % num) as reader:
with open(f'output-{num}.json') as reader:
values += json.load(reader)

values.sort()
Expand Down
17 changes: 8 additions & 9 deletions tests/plot.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def parse_data(infile):

if blocks.match(line):
try:
name = title.match(lines[index + 1]).group(1)
name = title.match(lines[index + 1])[1]
except:
index += 1
continue
Expand Down Expand Up @@ -90,18 +90,17 @@ def make_plot(data, action, save=False, show=False, limit=0.005):
ticks = ('Median', 'P90', 'P99')
index = (0, 1, 2)
names = list(data)
bars = []

for pos, (name, color) in enumerate(zip(names, colors)):
bars.append(ax.bar(
bars = [
ax.bar(
[val + pos * width for val in index],
[parse_timing(data[name][action][tick], limit) for tick in ticks],
width,
color=color,
))

)
for pos, (name, color) in enumerate(zip(names, colors))
]
ax.set_ylabel('Time (microseconds)')
ax.set_title('"%s" Time vs Percentile' % action)
ax.set_title(f'"{action}" Time vs Percentile')
ax.set_xticks([val + width * (len(data) / 2) for val in index])
ax.set_xticklabels(ticks)

Expand All @@ -118,7 +117,7 @@ def make_plot(data, action, save=False, show=False, limit=0.005):
plt.show()

if save:
plt.savefig('%s-%s.png' % (save, action), dpi=120, bbox_inches='tight')
plt.savefig(f'{save}-{action}.png', dpi=120, bbox_inches='tight')

plt.close()

Expand Down
27 changes: 11 additions & 16 deletions tests/stress_test_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,10 @@ def worker(queue, eviction_policy, processes, threads):
start = time.time()

try:
if action == 'set':
cache.set(key, value, expire=EXPIRE)
elif action == 'get':
if action == 'get':
result = cache.get(key)
elif action == 'set':
cache.set(key, value, expire=EXPIRE)
else:
assert action == 'delete'
cache.delete(key)
Expand All @@ -152,15 +152,15 @@ def worker(queue, eviction_policy, processes, threads):
delta = stop - start
timings[action].append(delta)
if miss:
timings[action + '-miss'].append(delta)
timings[f'{action}-miss'].append(delta)

queue.put(timings)

cache.close()


def dispatch(num, eviction_policy, processes, threads):
with open('input-%s.pkl' % num, 'rb') as reader:
with open(f'input-{num}.pkl', 'rb') as reader:
process_queue = pickle.load(reader)

thread_queues = [Queue.Queue() for _ in range(threads)]
Expand Down Expand Up @@ -194,7 +194,7 @@ def dispatch(num, eviction_policy, processes, threads):
for key in data:
timings[key].extend(data[key])

with open('output-%s.pkl' % num, 'wb') as writer:
with open(f'output-{num}.pkl', 'wb') as writer:
pickle.dump(timings, writer, protocol=2)


Expand All @@ -217,12 +217,7 @@ def stress_test(create=True, delete=True,
processes=1, threads=1):
shutil.rmtree('tmp', ignore_errors=True)

if processes == 1:
# Use threads.
func = threading.Thread
else:
func = mp.Process

func = threading.Thread if processes == 1 else mp.Process
subprocs = [
func(target=dispatch, args=(num, eviction_policy, processes, threads))
for num in range(processes)
Expand All @@ -236,7 +231,7 @@ def stress_test(create=True, delete=True,
process_queue[index % processes].append(ops)

for num in range(processes):
with open('input-%s.pkl' % num, 'wb') as writer:
with open(f'input-{num}.pkl', 'wb') as writer:
pickle.dump(process_queue[num], writer, protocol=2)

for process in subprocs:
Expand All @@ -254,15 +249,15 @@ def stress_test(create=True, delete=True,
timings = co.defaultdict(list)

for num in range(processes):
with open('output-%s.pkl' % num, 'rb') as reader:
with open(f'output-{num}.pkl', 'rb') as reader:
data = pickle.load(reader)
for key in data:
timings[key] += data[key]

if delete:
for num in range(processes):
os.remove('input-%s.pkl' % num)
os.remove('output-%s.pkl' % num)
os.remove(f'input-{num}.pkl')
os.remove(f'output-{num}.pkl')

display(eviction_policy, timings)

Expand Down
11 changes: 6 additions & 5 deletions tests/stress_test_deque_mp.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,11 +101,12 @@ def stress_rotate(deque):

def stress(seed, deque):
random.seed(seed)
for count in range(OPERATIONS):
if len(deque) > 100:
function = random.choice([stress_pop, stress_popleft])
else:
function = random.choice(functions)
for _ in range(OPERATIONS):
function = (
random.choice([stress_pop, stress_popleft])
if len(deque) > 100
else random.choice(functions)
)
function(deque)


Expand Down
Loading