fix bug with queue.empty not being reliable (deepspeedai#898)

mrwyattii · web-flow · commit cce62236a2c8 · 2024-05-01T11:11:56.000-07:00
See docs on `queue` for python multiprocessing. Specifically, `Because of multithreading/multiprocessing semantics, this is not reliable.`: https://docs.python.org/3/library/multiprocessing.html#multiprocessing.Queue.empty
diff --git a/benchmarks/inference/mii/src/client.py b/benchmarks/inference/mii/src/client.py
@@ -219,7 +219,7 @@ def _run_parallel(
 
     time.sleep(random.uniform(0, args.num_clients) * 0.01)
     try:
-        while not query_queue.empty():
+        while True:
             print(f"queue size: {query_queue.qsize()} ({pid})", flush=True)
             input_tokens, req_max_new_tokens = query_queue.get(timeout=1.0)
 
