添加校验函数

Boris-code · Boris-code · commit 2f4b55186abb · 2021-04-16T15:43:48.000+08:00
diff --git a/feapder/core/base_parser.py b/feapder/core/base_parser.py
@@ -26,25 +26,41 @@ def start_requests(self):
 
         pass
 
-    def parse(self, request, response):
+    def download_midware(self, request):
         """
-        @summary: 默认的回调函数
+        @summary: 下载中间件 可修改请求的一些参数
+        ---------
+        @param request:
+        ---------
+        @result: return request / None (不会修改原来的request)
+        """
+
+        pass
+
+    def validate(self, request, response):
+        """
+        @summary: 校验函数, 可用于校验response是否正确
+        若函数内抛出异常，则重试请求
+        若返回True 或 None，则进入解析函数
+        若返回False，则抛弃当前请求
+        可通过request.callback_name 区分不同的回调函数，编写不同的校验逻辑
         ---------
         @param request:
         @param response:
         ---------
-        @result:
+        @result: True / None / False
         """
 
         pass
 
-    def download_midware(self, request):
+    def parse(self, request, response):
         """
-        @summary: 下载中间件 可修改请求的一些参数
+        @summary: 默认的解析函数
         ---------
         @param request:
+        @param response:
         ---------
-        @result: return request / None (不会修改原来的request)
+        @result:
         """
 
         pass
@@ -108,12 +124,7 @@ class BatchParser(BaseParser):
     """
 
     def __init__(
-        self,
-        task_table,
-        batch_record_table,
-        task_state,
-        date_format,
-        mysqldb=None,
+        self, task_table, batch_record_table, task_state, date_format, mysqldb=None
     ):
         self._mysqldb = mysqldb or MysqlDB()  # mysqldb
 
diff --git a/feapder/core/parser_control.py b/feapder/core/parser_control.py
@@ -138,6 +138,10 @@ def deal_requests(self, requests):
                         else:
                             response = None
 
+                        # 校验
+                        if parser.validate(request, response) == False:
+                            continue
+
                         if request.callback:  # 如果有parser的回调函数，则用回调处理
                             callback_parser = (
                                 request.callback
@@ -473,6 +477,10 @@ def deal_requests(self, requests):
                         else:
                             response = None
 
+                        # 校验
+                        if parser.validate(request, response) == False:
+                            continue
+
                         if request.callback:  # 如果有parser的回调函数，则用回调处理
                             callback_parser = (
                                 request.callback
diff --git a/feapder/network/request.py b/feapder/network/request.py
@@ -223,6 +223,14 @@ def to_dict(self):
 
         return request_dict
 
+    @property
+    def callback_name(self):
+        return (
+            getattr(self.callback, "__name__")
+            if callable(self.callback)
+            else self.callback
+        )
+
     def get_response(self, save_cached=False):
         """
         获取带有selector功能的response
diff --git a/tests/air-spider/test_air_spider.py b/tests/air-spider/test_air_spider.py
@@ -25,6 +25,14 @@ def download_midware(self, request):
         # request.cookies = {}
         return request
 
+    def validate(self, request, response):
+        if response.status_code != 200:
+            raise Exception("response code not 200") # 重试
+
+        # if "哈哈" not in response.text:
+        #     return False # 或略当前请求
+
+
     def parse(self, request, response):
         print(response.bs4().title)
         print(response.xpath("//title").extract_first())
diff --git a/tests/spider/spiders/test_spider.py b/tests/spider/spiders/test_spider.py
@@ -13,7 +13,15 @@
 
 class TestSpider(feapder.Spider):
     def start_requests(self):
-        yield feapder.Request("https://www.baidu.com")
+        yield feapder.Request("https://www.baidu.com", callback=self.parse)
+
+    def validate(self, request, response):
+        print(request.callback_name)
+        if response.status_code != 200:
+            raise Exception("response code not 200")  # 重试
+
+        # if "哈哈" not in response.text:
+        #     return False # 或略当前请求
 
     def parse(self, request, response):
         title = response.xpath("//title/text()").extract_first()  # 取标题
