python3.6+scrapy下载一个网站全部产品网页的源码

scrapy startproject bsriceones

import scrapy from scrapy.linkextractors import LinkExtractor from scrapy.spiders import CrawlSpider, Rule import re,time class BselSpider(CrawlSpider): name = 'bs' allowed_domains = ['bsriceones.ga'] start_urls = ['http://www.bsriceones.ga/'] #规则：将需要点击的网址按规则一条一条写入，以下规则写入两条，一条是标题栏以及翻页的信息，一条是商品信息，并调用回调函数parse_item，这里来简单的，进行源码保存 rules=( Rule(LinkExtractor(allow=r'http://www.bsriceones.ga/[a-zA-Z0-9-]+/[a-zA-Z0-9-]+'), callback='parse_item', follow=True), #不需要回调函数 因为规则可以做到 Rule(LinkExtractor(allow=r'http://www.bsriceones.ga/[a-zA-Z0-9-]+/'),follow=True), #Rule(LinkExtractor(allow=r'http://www.bsriceones.ga/[a-zA-Z0-9-]+/.*page=\d+'), follow=True), ) def parse_item(self, response): html_name = re.findall(r'p-\d+',response.url)[0] print("-------------------------------------") print(response.url) print(html_name) print("-------------------------------------") with open("/samba/xww/py/pytest/bsriceones/bsriceones/bsriceones/html/"+html_name+".html",'wb') as f: f.write(response.body) 

#将这个改false ROBOTSTXT_OBEY = False #带上ua信息，可以在网页请求该网址的时候，将出现的ua信息copy下来，并带上referer参数，跟上你要爬的域名。 DEFAULT_REQUEST_HEADERS = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en', 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36', 'Referer':'http://www.bsriceones.ga', } 

pip install pymongo 

from pymongo import MongoClient client = MongoClient('mongodb://数据库地址:27017/') db=client.product

import request,json,random,datetime class BsspiderProxyIPLoadMiddleware(object): def __init__(self): self.proxy = "" self.expire_datetime=datetime.datetime.now()-datetime.timedelta(minutes=1) #self._get_proxyip() def _get_proxyip(self): #代理形式 ip：端口号 一个一行 f=open("proxy.txt") proxys= f.read().split("\n") p = random.sample(proxys,1)[0] print("proxy:",p) self.proxy = p self.expire_datetime = datetime.datetime.now()+datetime.timedelta(minutes=1) def _check_expire(self): if datetime.datetime.now() >= self.expire_datetime: self._get_proxyip() print("切换ip") def process_request(self,spider,request): self._check_expire() request.meta['proxy']="http://"+self.proxy 

DOWNLOADER_MIDDLEWARES = { 'middlewares所在文件的位置.middlewares.某某某spiderProxyIPLoadMiddleware': 543 }