diff --git a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py index 99227f9e38..d9ff9500f9 100644 --- a/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py +++ b/tools/documentation_crawler/documentation_crawler/commands/crawl_with_status.py @@ -18,6 +18,8 @@ class StatusCommand(Command): self.crawler_process.start() # Get exceptions quantity from crawler stat data stats = crawler.stats.get_stats() - if stats.get('spider_exceptions/Exception') or stats.get('downloader/exception_count'): + error_404 = 'downloader/response_status_count/404' + error_io = 'downloader/exception_type_count/exceptions.IOError' + if stats.get(error_404) or stats.get(error_io): # Return non-zero exit code if exceptions are contained self.exitcode = 1 diff --git a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py index e93425950a..6422e38b8b 100755 --- a/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py +++ b/tools/documentation_crawler/documentation_crawler/spiders/check_documentation.py @@ -81,12 +81,11 @@ class DocumentationSpider(scrapy.Spider): # type: (Any) -> Optional[Generator[Any, None, None]] if hasattr(failure.value, 'response') and failure.value.response: response = failure.value.response - if response.status in [500, 502, 503, 504, 429]: - self.log("Error! Please check link: {}".format(response), logging.ERROR) - return None + if response.status == 404: + raise Exception('Page not found: {}'.format(response)) if response.status == 405 and response.request.method == 'HEAD': # Method 'HEAD' not allowed, repeat request with 'GET' return self.retry_request_with_get(response.request) - raise Exception(failure.value.response) + self.log("Error! Please check link: {}".format(response), logging.ERROR) else: raise Exception(failure.value) diff --git a/tools/travis/backend b/tools/travis/backend index a89a722c52..5f23d4da80 100755 --- a/tools/travis/backend +++ b/tools/travis/backend @@ -10,6 +10,7 @@ set -x ./tools/test-management ./tools/test-migrations ./tools/test-run-dev +./tools/test-documentation # Some test suites disabled in CI for being flaky #./tools/test-queue-worker-reload -#./tools/test-documentation +