documentation_crawler: Reimplement crawl_with_status so it wraps crawl.

This way we inherit more of the upstream command’s behavior.
Importantly, this means we pass everything in `opts.spargs` to the
spider, not just `opts.spargs.skip_external`.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>

tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py

@@ -1,22 +1,20 @@
-from scrapy.commands.crawl import Command
+import optparse
-from scrapy.exceptions import UsageError
+from scrapy.crawler import Crawler
-from typing import List, Any
+from scrapy.commands import crawl
+from typing import List, Union
 
 
-class StatusCommand(Command):
+class Command(crawl.Command):
-    def run(self, args: List[str], opts: Any) -> None:
+    def run(self, args: List[str], opts: optparse.Values) -> None:
-        if len(args) < 1:
+        crawlers = []
-            raise UsageError()
+        real_create_crawler = self.crawler_process.create_crawler
-        elif len(args) > 1:
+
-            raise UsageError(
+        def create_crawler(crawler_or_spidercls: Union[Crawler, str]) -> Crawler:
-                "running 'scrapy crawl' with more than one spider is no longer supported")
+            crawler = real_create_crawler(crawler_or_spidercls)
-        spname = args[0]
+            crawlers.append(crawler)
-        if len(vars(opts)['spargs']) > 0:
+            return crawler
-            skip_external = vars(opts)['spargs']['skip_external']
+
-        else:
+        self.crawler_process.create_crawler = create_crawler
-            skip_external = None
+        super().run(args, opts)
-        crawler = self.crawler_process.create_crawler(spname)
+        if any(crawler.stats.get_value("log_count/ERROR") for crawler in crawlers):
-        self.crawler_process.crawl(crawler, skip_external=skip_external)
-        self.crawler_process.start()
-        if crawler.stats.get_value("log_count/ERROR"):
             self.exitcode = 1