paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-03 21:43:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

tools: Upgrade scrapy's settings.py to the latest version.

Browse Source 
Fixes a warning caused by using the deprecated `2.6` value
of the `REQUEST_FINGERPRINTER_IMPLEMENTATION` setting.

Upgrades `settings.py` to what would have been generated
by starting the documentation_crawler project using the
`startproject` command of scrapy v2.7.
This commit is contained in:
David Rosa
2022-11-09 12:59:36 -05:00
committed by Anders Kaseorg
parent 0fa7b8e1ae
commit 233e690fb4
1 changed files with 18 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
tools/documentation_crawler/documentation_crawler/settings.py
View File

@@ -3,9 +3,9 @@
# For simplicity, this file contains only settings considered important or # For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation: # commonly used. You can find more settings consulting the documentation:
# #
# http://doc.scrapy.org/en/latest/topics/settings.html # https://docs.scrapy.org/en/latest/topics/settings.html
# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html # https://docs.scrapy.org/en/latest/topics/spider-middleware.html
BOT_NAME = "documentation_crawler" BOT_NAME = "documentation_crawler"
@@ -30,7 +30,7 @@ ROBOTSTXT_OBEY = False
# CONCURRENT_REQUESTS = 32 # CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0) # Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs # See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3 # DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of: # The download delay setting will honor only one of:
@@ -40,7 +40,7 @@ ROBOTSTXT_OBEY = False
# Disable cookies (enabled by default) # Disable cookies (enabled by default)
# COOKIES_ENABLED = False # COOKIES_ENABLED = False
# Disable telnet console (enabled by default) # Disable Telnet Console (enabled by default)
# TELNETCONSOLE_ENABLED = False # TELNETCONSOLE_ENABLED = False
# Override the default request headers: # Override the default request headers:
@@ -50,31 +50,31 @@ ROBOTSTXT_OBEY = False
# } # }
# Enable or disable spider middlewares # Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html # See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
# SPIDER_MIDDLEWARES = { # SPIDER_MIDDLEWARES = {
# 'documentation_crawler.middlewares.MyCustomSpiderMiddleware': 543, # 'documentation_crawler.middlewares.DocumentationCrawlerSpiderMiddleware': 543,
# } # }
# Enable or disable downloader middlewares # Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = { # DOWNLOADER_MIDDLEWARES = {
# 'documentation_crawler.middlewares.MyCustomDownloaderMiddleware': 543, # 'documentation_crawler.middlewares.DocumentationCrawlerDownloaderMiddleware': 543,
# } # }
# Enable or disable extensions # Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html # See https://docs.scrapy.org/en/latest/topics/extensions.html
# EXTENSIONS = { # EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None, # 'scrapy.extensions.telnet.TelnetConsole': None,
# } # }
# Configure item pipelines # Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html # See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# ITEM_PIPELINES = { # ITEM_PIPELINES = {
# 'documentation_crawler.pipelines.SomePipeline': 300, # 'documentation_crawler.pipelines.DocumentationCrawlerPipeline': 300,
# } # }
# Enable and configure the AutoThrottle extension (disabled by default) # Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
# AUTOTHROTTLE_ENABLED = True # AUTOTHROTTLE_ENABLED = True
# The initial download delay # The initial download delay
# AUTOTHROTTLE_START_DELAY = 5 # AUTOTHROTTLE_START_DELAY = 5
@@ -87,9 +87,13 @@ ROBOTSTXT_OBEY = False
# AUTOTHROTTLE_DEBUG = False # AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default) # Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
# HTTPCACHE_ENABLED = True # HTTPCACHE_ENABLED = True
# HTTPCACHE_EXPIRATION_SECS = 0 # HTTPCACHE_EXPIRATION_SECS = 0
# HTTPCACHE_DIR = 'httpcache' # HTTPCACHE_DIR = 'httpcache'
# HTTPCACHE_IGNORE_HTTP_CODES = [] # HTTPCACHE_IGNORE_HTTP_CODES = []
# HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"