I have a similar problem with that. I have hundreds of spiders and most settings of them are the same, several are customized. Here is my solution.
First, make a spider_settings.py file:
project_name = 'xxx'
downloader_map = { # you can list all your downloadmiddlewares here with a alias
'json_check': (201, 'xy_spider.middlewares.MustJsonDecodeMiddleware'),
'abuyun': (202, 'xy_spider.middlewares.AbuyunProxyMiddleware'),
'charset_change': (203, 'xy_spider.middlewares.CharsetSwitchDownloaderMiddleware'),
}
pipeline_map = { # 0-1000
'print': (301, 'xy_spider.pipelines.printPipeline'),
'merge': (1, 'xy_spider.pipelines.MergePagesPipelineForNews')
}
class CommonSettings:
def __init__(self, **kwargs):
kwargs = kwargs or {}
_downloader_mids = {'xy_spider.middlewares.XySpiderDownloaderMiddleware': 200}
for k, v in downloader_map.items():
if k in kwargs and kwargs[k]:
_downloader_mids[v[1]] = v[0]
_pipeline = { # here are the pipelines necessary for all spiders
f'projects.{project_name}.pipelines.MySQLPipeline': 300,
f'projects.{project_name}.pipelines.MysqlQueuePipeline': 901,
}
for k, v in pipeline_map.items():
if k in kwargs and kwargs[k]:
_pipeline[v[1]] = v[0]
timeout = kwargs.get('timeout', kwargs.get('DOWNLOAD_TIMEOUT'))
self._settings = {
'LOG_LEVEL': 'INFO',
'SCHEDULER': 'xy_spider.redis.expire_scheduler.ExpireScheduler',
'DUPEFILTER_CLASS': 'xy_spider.redis.expire_dupefilter.ExpireDupeFilter',
'DUPEFILTER_EXPIRE_DAYS': int(kwargs.get('expire', kwargs.get('DUPEFILTER_EXPIRE_DAYS', 7))),
'SCHEDULER_PERSIST': True,
'SCHEDULER_FLUSH_ON_START': bool(kwargs.get('get_all', True)),
'DOWNLOAD_DELAY': int(kwargs.get('delay', kwargs.get('DOWNLOAD_DELAY', 1))),
'CONCURRENT_REQUESTS': int(kwargs.get('thread', kwargs.get('CONCURRENT_REQUESTS', 10))),
"SPIDER_MIDDLEWARES": {
'xy_spider.middlewares.RandomParamFilterSpiderMiddleware': 200,
},
"DOWNLOADER_MIDDLEWARES": _downloader_mids,
'ITEM_PIPELINES': _pipeline,
}
if timeout is not None:
self._settings['DOWNLOAD_TIMEOUT'] = int(timeout) or 20
def __call__(self):
return self._settings
Second, you can now customize your settings easily by create an instance of CommonSettings() in your spider class like:
class TestSpider(scrapy.Spider):
name = "test"
is_test = not on_server
_settings = {
'delay': 1,
'json_check': 1,
'timeout': 10,
'thread': 1,
}
if is_test:
_settings['print'] = 1
else:
_settings['get_all'] = 0
custom_settings = CommonSettings(**_settings)()