paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-06 15:03:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

docs test: Verify zulip/zulip file and directory links locally.

Browse Source
This commit is contained in:
Vishnu KS
2020-11-03 19:07:24 +05:30
committed by Tim Abbott
parent cba292b4bb
commit d6743da139
4 changed files with 23 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tools/documentation_crawler/documentation_crawler/spiders/common/spiders.py
View File

@@ -1,4 +1,5 @@
import json
import os
import re
from typing import Callable, Iterator, List, Optional, Union
@@ -42,6 +43,10 @@ VNU_IGNORE = [
]
VNU_IGNORE_REGEX = re.compile(r'|'.join(VNU_IGNORE))
DEPLOY_ROOT = os.path.abspath(os.path.join(__file__, "../../../../../.."))
GITHUB_FILE_URL_PREFIX = "https://github.com/zulip/zulip/blob/master"
GITHUB_DIRECTORY_URL_PREFIX = "https://github.com/zulip/zulip/tree/master"
class BaseDocumentationSpider(scrapy.Spider):
name: Optional[str] = None
@@ -69,7 +74,7 @@ class BaseDocumentationSpider(scrapy.Spider):
# homepage, there's no need to check those (which can
# cause errors when chat.zulip.org is being updated).
return True
if "zulip.readthedocs" in url or "zulip.com" in url or "zulip.org" in url:
if "zulip.readthedocs" in url or "zulip.com" in url or "zulip.org" in url or "github.com/zulip/zulip/" in url:
# We want CI to check any links to Zulip sites.
return False
if (len(url) > 4 and url[:4] == "file") or ("localhost" in url):
@@ -126,6 +131,20 @@ class BaseDocumentationSpider(scrapy.Spider):
if self._is_external_url(url):
callback = self.check_existing
method = 'HEAD'
if url.startswith(GITHUB_FILE_URL_PREFIX):
file_path = url.replace(GITHUB_FILE_URL_PREFIX, DEPLOY_ROOT)
hash_index = file_path.find("#")
if hash_index != -1:
file_path = file_path[:hash_index]
if not os.path.isfile(file_path):
self.logger.error("There is no local file associated with the GitHub URL: %s", url)
return
elif url.startswith(GITHUB_DIRECTORY_URL_PREFIX):
dir_path = url.replace(GITHUB_DIRECTORY_URL_PREFIX, DEPLOY_ROOT)
if not os.path.isdir(dir_path):
self.logger.error("There is no local directory associated with the GitHub URL: %s", url)
return
elif '#' in url:
dont_filter = True
callback = self.check_fragment