下载中间件拦截请求
需求
在《行百里者半九十 —— scrapy 框架(6)》一文中我们介绍了下载中间件的作用,并演示了其中拦截响应的代码实现。
现在我们来试着实现拦截请求的代码实现,也就是UA池和代理池的实现。因为免费 IP 总是失效,所以在这里只提供中间件部分的代码实现,不提供运行结果。
正因为此,代码可能有所疏漏,还望各位看官海涵。
中间件部分代码实现
# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
from scrapy import signals
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter
# class MidproSpiderMiddleware:
# # Not all methods need to be defined. If a method is not defined,
# # scrapy acts as if the spider middleware does not modify the
# # passed objects.
#
# @classmethod
# def from_crawler(cls, crawler):
# # This method is used by Scrapy to create your spiders.
# s = cls()
# crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
# return s
#
# def process_spider_input(self, response, spider):
# # Called for each response that goes through the spider
# # middleware and into the spider.
#
# # Should return None or raise an exception.
# return None
#
# def process_spider_output(self, response, result, spider):
# # Called with the results returned from the Spider, after
# # it has processed the response.
#
# # Must return an iterable of Request, or item objects.
# for i in result:
# yield i
#
# def process_spider_exception(self, response, exception, spider):
# # Called when a spider or process_spider_input() method
# # (from other spider middleware) raises an exception.
#
# # Should return either None or an iterable of Request or item objects.
# pass
#
# def process_start_requests(self, start_requests, spider):
# # Called with the start requests of the spider, and works
# # similarly to the process_spider_output() method, except
# # that it doesn’t have a response associated.
#
# # Must return only requests (not items).
# for r in start_requests:
# yield r
#
# def spider_opened(self, spider):
# spider.logger.info('Spider opened: %s' % spider.name)
import random
class MidproDownloaderMiddleware:
# 代理池
proxy_http = []
proxy_https = []
# 拦截请求
def process_request(self, request, spider):
# UA伪装
user_agent_list = [] # UA池
request.headers["User-Agent"] = random.choice(user_agent_list)
return None
# 拦截所有响应
def process_response(self, request, response, spider):
# Called with the response returned from the downloader.
# Must either;
# - return a Response object
# - return a Request object
# - or raise IgnoreRequest
return response
# 拦截所有异常
def process_exception(self, request, exception, spider):
if request.url.split(":")[0] == "http":
request.meta["proxy"] = "http://" + random.choice(self.proxy_http)
else:
request.meta["proxy"] = "https://" + random.choice(self.proxy_https)
return request # 将修正之后的请求对象重新进行请求发送