test-documentation: Add --skip-external-links option.

This option causes test-documentation to only verify internal links
that we control and are important to be correct.  This prevents
test-documentation from flaking in CI due to issues with the dozens of
third-party blocks that we link to from various parts of our
documentation.

Tweaked by tabbott for comment clarity, and to also include
github.com/zulip links.

Fixes #10942.

tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py

@@ -34,6 +34,7 @@ class BaseDocumentationSpider(scrapy.Spider):
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.has_error = False
+        self.skip_external = kwargs.get('skip_external', None)
 
     def _set_error_state(self) -> None:
         self.has_error = True
@@ -47,6 +48,18 @@ class BaseDocumentationSpider(scrapy.Spider):
     def check_existing(self, response: Any) -> None:
         self.log(response)
 
+    def _is_external_link(self, url: str) -> bool:
+        if "zulip.readthedocs" in url or "zulipchat.com" in url or "zulip.org" in url:
+            # We want CI to check any links to Zulip sites.
+            return False
+        if (len(url) > 4 and url[:4] == "file") or ("localhost" in url):
+            # We also want CI to check any links to built documentation.
+            return False
+        if 'github.com/zulip' in url:
+            # Finally, links to our own GitHub organization should always work.
+            return False
+        return True
+
     def check_permalink(self, response: Any) -> None:
         self.log(response)
         xpath_template = "//*[@id='{permalink}' or @name='{permalink}']"
@@ -74,6 +87,9 @@ class BaseDocumentationSpider(scrapy.Spider):
             elif '#' in link.url:
                 dont_filter = True
                 callback = self.check_permalink
+            if (self.skip_external is not None):   # checks if flag is set to skip external link check.
+                if (self._is_external_link(link.url)):
+                    continue
             yield Request(link.url, method=method, callback=callback, dont_filter=dont_filter,
                           errback=self.error_callback)