Files
zulip/zerver/lib/parallel.py
Anders Kaseorg fead14951c python: Convert assignment type annotations to Python 3.6 style.
This commit was split by tabbott; this piece covers the vast majority
of files in Zulip, but excludes scripts/, tools/, and puppet/ to help
ensure we at least show the right error messages for Xenial systems.

We can likely further refine the remaining pieces with some testing.

Generated by com2ann, with whitespace fixes and various manual fixes
for runtime issues:

-    invoiced_through: Optional[LicenseLedger] = models.ForeignKey(
+    invoiced_through: Optional["LicenseLedger"] = models.ForeignKey(

-_apns_client: Optional[APNsClient] = None
+_apns_client: Optional["APNsClient"] = None

-    notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
-    signup_notifications_stream: Optional[Stream] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+    notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)
+    signup_notifications_stream: Optional["Stream"] = models.ForeignKey('Stream', related_name='+', null=True, blank=True, on_delete=CASCADE)

-    author: Optional[UserProfile] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)
+    author: Optional["UserProfile"] = models.ForeignKey('UserProfile', blank=True, null=True, on_delete=CASCADE)

-    bot_owner: Optional[UserProfile] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)
+    bot_owner: Optional["UserProfile"] = models.ForeignKey('self', null=True, on_delete=models.SET_NULL)

-    default_sending_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
-    default_events_register_stream: Optional[Stream] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+    default_sending_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)
+    default_events_register_stream: Optional["Stream"] = models.ForeignKey('zerver.Stream', null=True, related_name='+', on_delete=CASCADE)

-descriptors_by_handler_id: Dict[int, ClientDescriptor] = {}
+descriptors_by_handler_id: Dict[int, "ClientDescriptor"] = {}

-worker_classes: Dict[str, Type[QueueProcessingWorker]] = {}
-queues: Dict[str, Dict[str, Type[QueueProcessingWorker]]] = {}
+worker_classes: Dict[str, Type["QueueProcessingWorker"]] = {}
+queues: Dict[str, Dict[str, Type["QueueProcessingWorker"]]] = {}

-AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional[LDAPSearch] = None
+AUTH_LDAP_REVERSE_EMAIL_SEARCH: Optional["LDAPSearch"] = None

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
2020-04-22 11:02:32 -07:00

76 lines
2.0 KiB
Python

from typing import Dict, Iterable, Tuple, Callable, TypeVar, Iterator
import os
import pty
import sys
import errno
JobData = TypeVar('JobData')
def run_parallel(job: Callable[[JobData], int],
data: Iterable[JobData],
threads: int=6) -> Iterator[Tuple[int, JobData]]:
pids: Dict[int, JobData] = {}
def wait_for_one() -> Tuple[int, JobData]:
while True:
try:
(pid, status) = os.wait()
return status, pids.pop(pid)
except KeyError:
pass
for item in data:
pid = os.fork()
if pid == 0:
sys.stdin.close()
try:
os.close(pty.STDIN_FILENO)
except OSError as e:
if e.errno != errno.EBADF:
raise
sys.stdin = open("/dev/null")
os._exit(job(item))
pids[pid] = item
threads = threads - 1
if threads == 0:
(status, item) = wait_for_one()
threads += 1
yield (status, item)
if status != 0:
# Stop if any error occurred
break
while True:
try:
(status, item) = wait_for_one()
yield (status, item)
except OSError as e:
if e.errno == errno.ECHILD:
break
else:
raise
if __name__ == "__main__":
# run some unit tests
import time
jobs = [10, 19, 18, 6, 14, 12, 8, 2, 1, 13, 3, 17, 9, 11, 5, 16, 7, 15, 4]
expected_output = [6, 10, 12, 2, 1, 14, 8, 3, 18, 19, 5, 9, 13, 11, 4, 7, 17, 16, 15]
def wait_and_print(x: int) -> int:
time.sleep(x * 0.1)
return 0
output = []
for (status, job) in run_parallel(wait_and_print, jobs):
output.append(job)
if output == expected_output:
print("Successfully passed test!")
else:
print("Failed test!")
print(jobs)
print(expected_output)
print(output)