scrapy-redis写项目备忘

爬虫文件：

from scrapy.spiders import Rule
from scrapy.linkextractors import LinkExtractor
from scrapy_redis.spiders import RedisCrawlSpider
class MyCrawler(RedisCrawlSpider):
    name = 'mycrawler_redis'
    redis_key = 'mycrawler:start_urls'
#规则
    rules = (
        # follow all links
        Rule(LinkExtractor(), callback='parse_page', follow=True),
    )

#重点是allowed_domains后边弄成数组list
    def __init__(self, *args, **kwargs):
        # Dynamically define the allowed domains list.
        domain = kwargs.pop('domain', '')
        self.allowed_domains =list(filter(None, domain.split(',')))
        super(MyCrawler, self).__init__(*args, **kwargs)

    def parse_page(self, response):
        return {
            'name': response.css('title::text').extract_first(),
            'url': response.url,
        }

在setting文件中加入如下代码：

REDIS_URL = 'redis://root:@127.0.0.1:6379'
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"
DUPEFILTER_DEBUG =True
SCHEDULER = "scrapy_redis.scheduler.Scheduler"
SCHEDULER_PERSIST = True
SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.SpiderPriorityQueue'