docs test: Verify zulip/zulip file and directory links locally.

docs/testing/linters.md

@@ -130,7 +130,7 @@ script performs several lint checks in parallel by forking out subprocesses.
 
 Note that our project does custom regex-based checks on the code, and we
 also customize how we call pyflakes and pycodestyle (pep8).  The code for these
-types of checks mostly lives [here](https://github.com/zulip/zulip/blob/master/tools/linter_lib).
+types of checks mostly lives [here](https://github.com/zulip/zulip/tree/master/tools/linter_lib).
 
 ### Special options
 

docs/tutorials/writing-views.md

@@ -98,7 +98,7 @@ def home(request: HttpRequest) -> HttpResponse:
 ### Writing a template
 
 Templates for the main website are found in
-[templates/zerver/app](https://github.com/zulip/zulip/blob/master/templates/zerver/app).
+[templates/zerver/app](https://github.com/zulip/zulip/tree/master/templates/zerver/app).
 
 
 ## Writing API REST endpoints

templates/zerver/api/integrations-overview.md

@@ -69,5 +69,5 @@ us](/help/contact-support) and we'll see what we can do.
   built on top of this API, so it can do anything a human user can do. Most
   but not all of the endpoints are documented on this site; if you need
   something that isn't there check out Zulip's
-  [REST endpoints](https://github.com/zulip/zulip/tree/master/zproject/urls.py)
+  [REST endpoints](https://github.com/zulip/zulip/blob/master/zproject/urls.py)
   or [contact us](/help/contact-support) and we'll help you out.

tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py

@@ -1,4 +1,5 @@
 import json
+import os
 import re
 from typing import Callable, Iterator, List, Optional, Union
 
@@ -42,6 +43,10 @@ VNU_IGNORE = [
 ]
 VNU_IGNORE_REGEX = re.compile(r'|'.join(VNU_IGNORE))
 
+DEPLOY_ROOT = os.path.abspath(os.path.join(__file__, "../../../../../.."))
+
+GITHUB_FILE_URL_PREFIX = "https://github.com/zulip/zulip/blob/master"
+GITHUB_DIRECTORY_URL_PREFIX = "https://github.com/zulip/zulip/tree/master"
 
 class BaseDocumentationSpider(scrapy.Spider):
     name: Optional[str] = None
@@ -69,7 +74,7 @@ class BaseDocumentationSpider(scrapy.Spider):
             # homepage, there's no need to check those (which can
             # cause errors when chat.zulip.org is being updated).
             return True
-        if "zulip.readthedocs" in url or "zulip.com" in url or "zulip.org" in url:
+        if "zulip.readthedocs" in url or "zulip.com" in url or "zulip.org" in url or "github.com/zulip/zulip/" in url:
             # We want CI to check any links to Zulip sites.
             return False
         if (len(url) > 4 and url[:4] == "file") or ("localhost" in url):
@@ -126,6 +131,20 @@ class BaseDocumentationSpider(scrapy.Spider):
         if self._is_external_url(url):
             callback = self.check_existing
             method = 'HEAD'
+
+            if url.startswith(GITHUB_FILE_URL_PREFIX):
+                file_path = url.replace(GITHUB_FILE_URL_PREFIX, DEPLOY_ROOT)
+                hash_index = file_path.find("#")
+                if hash_index != -1:
+                    file_path = file_path[:hash_index]
+                if not os.path.isfile(file_path):
+                    self.logger.error("There is no local file associated with the GitHub URL: %s", url)
+                return
+            elif url.startswith(GITHUB_DIRECTORY_URL_PREFIX):
+                dir_path = url.replace(GITHUB_DIRECTORY_URL_PREFIX, DEPLOY_ROOT)
+                if not os.path.isdir(dir_path):
+                    self.logger.error("There is no local directory associated with the GitHub URL: %s", url)
+                return
         elif '#' in url:
             dont_filter = True
             callback = self.check_fragment