documentation_crawler: Set a 15 second download timeout.

By default it just hangs for as long as the server keeps the
connection open.

Signed-off-by: Anders Kaseorg <anders@zulipchat.com>
This commit is contained in:
Anders Kaseorg
2019-05-30 00:59:23 -07:00
committed by Tim Abbott
parent c59747360a
commit 4f28b88746

View File

@@ -15,6 +15,7 @@ SPIDER_MODULES = ['documentation_crawler.spiders']
NEWSPIDER_MODULE = 'documentation_crawler.spiders'
COMMANDS_MODULE = 'documentation_crawler.commands'
LOG_LEVEL = 'WARNING'
DOWNLOAD_TIMEOUT = 15
# Crawl responsibly by identifying yourself (and your website) on the user-agent