Skip to content

Commit 4c552cf

Browse files
fix: green threads throw exceptions properly (seung-lab#34)
* fix: green threads throw exceptions properly fix: green threads count progress like regular threads * chore: ensure moto has all deps
1 parent 3ac7429 commit 4c552cf

3 files changed

Lines changed: 32 additions & 13 deletions

File tree

‎requirements_dev.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
moto
1+
moto[all]
22
pytest

‎taskqueue/scheduler.py‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
def schedule_threaded_jobs(
1111
fns, concurrency=DEFAULT_THREADS,
12-
progress=None, total=None, batch_size=1
12+
progress=None, total=None
1313
):
1414

1515
if total is None:
@@ -38,7 +38,7 @@ def realupdatefn(iface):
3838

3939
def schedule_green_jobs(
4040
fns, concurrency=DEFAULT_THREADS,
41-
progress=None, total=None, batch_size=1
41+
progress=None, total=None
4242
):
4343
import gevent.pool
4444

@@ -51,29 +51,41 @@ def schedule_green_jobs(
5151
desc = progress if isinstance(progress, STRING_TYPES) else None
5252

5353
pbar = tqdm(total=total, desc=desc, disable=(not progress))
54+
5455
results = []
55-
56+
exceptions = []
57+
58+
def add_exception(greenlet):
59+
nonlocal exceptions
60+
try:
61+
greenlet.get()
62+
except Exception as err:
63+
exceptions.append(err)
64+
5665
def updatefn(fn):
5766
def realupdatefn():
58-
res = fn()
59-
pbar.update(batch_size)
60-
results.append(res)
67+
ct = fn()
68+
pbar.update(ct)
69+
results.append(ct) # cPython list append is thread safe
6170
return realupdatefn
6271

6372
pool = gevent.pool.Pool(concurrency)
6473
for fn in fns:
65-
pool.spawn( updatefn(fn) )
74+
greenlet = pool.spawn( updatefn(fn) )
75+
greenlet.link_exception(add_exception)
6676

6777
pool.join()
6878
pool.kill()
6979
pbar.close()
7080

81+
if exceptions:
82+
raise_multiple(exceptions)
83+
7184
return results
7285

7386
def schedule_jobs(
7487
fns, concurrency=DEFAULT_THREADS,
75-
progress=None, total=None, green=False,
76-
batch_size=1
88+
progress=None, total=None, green=False
7789
):
7890
"""
7991
Given a list of functions, execute them concurrently until
@@ -93,8 +105,8 @@ def schedule_jobs(
93105
return [ fn() for fn in tqdm(fns, disable=(not progress or total == 1), desc=progress) ]
94106

95107
if green:
96-
return schedule_green_jobs(fns, concurrency, progress, total, batch_size)
108+
return schedule_green_jobs(fns, concurrency, progress, total)
97109

98-
return schedule_threaded_jobs(fns, concurrency, progress, total, batch_size)
110+
return schedule_threaded_jobs(fns, concurrency, progress, total)
99111

100112

‎taskqueue/taskqueue.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,6 @@ def insertfn(batch):
191191
progress=('Inserting' if self.progress else False),
192192
total=total,
193193
green=self.green,
194-
batch_size=batch_size,
195194
)
196195
cts = sum(cts)
197196

@@ -584,5 +583,13 @@ def capturing_soloprocess_upload(*args, **kwargs):
584583

585584
return ct
586585

586+
# c/o https://stackoverflow.com/questions/12826291/raise-two-errors-at-the-same-time
587+
def raise_multiple(errors):
588+
if not errors:
589+
return
590+
try:
591+
raise errors.pop()
592+
finally:
593+
raise_multiple(errors)
587594

588595

0 commit comments

Comments
 (0)