test-help-documentation: Skips checking external links.

This option causes test-help-documentation to verify internal links only.
This prevents documentation from flaking in CI due to links from
external websites having failed in the documentation.

Based on conversation:
https://chat.zulip.org/#narrow/stream/43-automated-testing/subject/quora.20404/near/674997

tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py

@@ -11,9 +11,12 @@ class StatusCommand(Command):
             raise UsageError(
                 "running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
-
+        if len(vars(opts)['spargs']) > 0:
+            skip_external = vars(opts)['spargs']['skip_external']
+        else:
+            skip_external = None
         crawler = self.crawler_process.create_crawler(spname)
-        self.crawler_process.crawl(crawler)
+        self.crawler_process.crawl(crawler, skip_external=skip_external)
         self.crawler_process.start()
         # Get exceptions quantity from crawler stat data
 

tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py

@@ -87,7 +87,7 @@ class BaseDocumentationSpider(scrapy.Spider):
             elif '#' in link.url:
                 dont_filter = True
                 callback = self.check_permalink
-            if (self.skip_external is not None):   # checks if flag is set to skip external link check.
+            if self.skip_external:
                 if (self._is_external_link(link.url)):
                     continue
             yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,