一文教你使用scrapy-redis组件

pip install scrapy-redis-cluster # 安装模块
pip install scrapy-redis-cluster==0.4 # 安装模块时指定版本
pip install --upgrade scrapy-redis-cluster # 升级模块版本

d:\scrapy>scrapy startproject jd_book

# -*- coding: utf-8 -*-

import scrapy

class JdBookItem(scrapy.Item):
  b_cate = scrapy.Field() # 图书所属一级分类名称
  s_cate = scrapy.Field() # 图书所属二级分类名称
  s_href = scrapy.Field() # 图书所属二级分类地址
  book_name = scrapy.Field() # 名称
  book_img = scrapy.Field() # 封面图片地址
  book_author = scrapy.Field() # 作者
  book_press = scrapy.Field() # 出版社
  book_publish_date = scrapy.Field() # 出版日期
  book_sku = scrapy.Field() # 商品编号
  book_price = scrapy.Field() # 价格

d:\scrapy\jd_book>cd jd_book
d:\scrapy\jd_book>scrapy genspider jdbook jd.com

# -*- coding: utf-8 -*-
import scrapy
import json
import urllib
from copy import deepcopy
from jd_book.items import JdBookItem

class JdbookSpider(scrapy.Spider):
  name = 'jdbook'
  allowed_domains = ['jd.com','3.cn']
  start_urls = ['https://book.jd.com/booksort.html']

  def parse(self, response): # 处理图书分类页
    dt_list = response.xpath("//div[@class='mc']/dl/dt") # 提取一级分类元素
    for dt in dt_list:
      item = JdBookItem()
      item["b_cate"] = dt.xpath("./a/text()").extract_first() # 提取一级分类名称
      em_list = dt.xpath("./following-sibling::dd[1]/em") # 提取二级分类元素
      for em in em_list:
        item["s_cate"] = em.xpath("./a/text()").extract_first() # 提取二级分类名称
        item["s_href"] = em.xpath("./a/@href").extract_first() # 提取二级分类地址
        if item["s_href"] is not None:
          item['s_href'] = "https:" + item['s_href'] # 补全二级分类地址
          yield scrapy.Request(item['s_href'], callback=self.parse_book_list, meta={"item": deepcopy(item)})

  def parse_book_list(self, response): # 处理二级分类下图书列表页
    item = response.meta['item']
    li_list = response.xpath("//div[@id='plist']/ul/li") # 提取所有的图书元素
    for li in li_list:
      item["book_img"] = li.xpath(".//div[@class='p-img']//img/@data-lazy-img").extract_first()
      if item["book_img"] is None:
        item["book_img"] = li.xpath(".//div[@class='p-img']//img/@src").extract_first()
      if item["book_img"] is not None:
        item["book_img"] = "https:"+item["book_img"]
      item["book_name"] = li.xpath(".//div[@class='p-name']/a/em/text()").extract_first().strip()
      item["book_author"] = li.xpath(".//span[@class='author_type_1']/a/text()").extract()
      item["book_press"] = li.xpath(".//span[@class='p-bi-store']/a/@title").extract_first()
      item["book_publish_date"] = li.xpath(".//span[@class='p-bi-date']/text()").extract_first().strip()
      item["book_sku"] = li.xpath("./div/@data-sku").extract_first()
      price_url = "https://p.3.cn/prices/mgets?skuIds=j_{}".format(item["book_sku"]) # 提取图书价格请求地址
      yield scrapy.Request(price_url, callback=self.parse_book_price, meta={"item": deepcopy(item)})

    # 提取列表页下一页地址
    next_url = response.xpath("//a[@class='pn-next']/@href").extract_first()
    if next_url is not None:
      next_url = urllib.parse.urljoin(response.url, next_url)
      # yield scrapy.Request(next_url,callback=self.parse_book_list,meta={"item":item})

  def parse_book_price(self, response):
    item = response.meta['item']
    item["book_price"] = json.loads(response.body.decode())[0]["op"]
    yield item

# -*- coding: utf-8 -*-

BOT_NAME = 'jd_book'

SPIDER_MODULES = ['jd_book.spiders']
NEWSPIDER_MODULE = 'jd_book.spiders'

# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False


######################################################
##############下面是Scrapy-Redis相关配置################
######################################################

# 指定Redis的主机名和端口
REDIS_HOST = 'localhost'
REDIS_PORT = 6379

# 调度器启用Redis存储Requests队列
SCHEDULER = "scrapy_redis.scheduler.Scheduler"

# 确保所有的爬虫实例使用Redis进行重复过滤
DUPEFILTER_CLASS = "scrapy_redis.dupefilter.RFPDupeFilter"

# 将Requests队列持久化到Redis，可支持暂停或重启爬虫
SCHEDULER_PERSIST = True

# Requests的调度策略，默认优先级队列
SCHEDULER_QUEUE_CLASS = 'scrapy_redis.queue.PriorityQueue'

# 将爬取到的items保存到Redis 以便进行后续处理
ITEM_PIPELINES = {
  'scrapy_redis.pipelines.RedisPipeline': 300
}

d:\scrapy\jd_book>scrapy crawl jdbook