浏览器渲染支持自定义参数

Boris-code · Boris-code · commit 5da48bbc0cdf · 2021-05-07T11:52:26.000+08:00
diff --git a/feapder/setting.py b/feapder/setting.py
@@ -53,8 +53,10 @@
 SPIDER_SLEEP_TIME = 0  # 下载时间间隔（解析完一个response后休眠时间）
 SPIDER_TASK_COUNT = 1  # 每个parser从内存队列中获取任务的数量
 SPIDER_MAX_RETRY_TIMES = 100  # 每个请求最大重试次数
-SPIDER_AUTO_START_REQUESTS = True # 是否主动执行添加 设置为False 需要手动调用start_monitor_task，适用于多进程情况下
-AUTO_STOP_WHEN_SPIDER_DONE = True # 爬虫是否自动结束
+SPIDER_AUTO_START_REQUESTS = (
+    True
+)  # 是否主动执行添加 设置为False 需要手动调用start_monitor_task，适用于多进程情况下
+AUTO_STOP_WHEN_SPIDER_DONE = True  # 爬虫是否自动结束
 
 # 浏览器渲染
 WEBDRIVER = dict(
@@ -68,7 +70,7 @@
     window_size=(1024, 800),  # 窗口大小
     executable_path=None,  # 浏览器路径，默认为默认路径
     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
-    custom_argument=['--ignore-certificate-errors']  # 自定义浏览器渲染参数
+    custom_argument=["--ignore-certificate-errors"],  # 自定义浏览器渲染参数
 )
 
 # 爬虫启动时，重新抓取失败的requests
diff --git a/feapder/templates/project_template/setting.py b/feapder/templates/project_template/setting.py
@@ -56,6 +56,7 @@
     window_size=(1024, 800),  # 窗口大小
     executable_path=None,  # 浏览器路径，默认为默认路径
     render_time=0,  # 渲染时长，即打开网页等待指定时间后再获取源码
+    custom_argument=["--ignore-certificate-errors"],  # 自定义浏览器渲染参数
 )
 
 # 爬虫启动时，重新抓取失败的requests
diff --git a/feapder/utils/webdriver.py b/feapder/utils/webdriver.py
@@ -27,17 +27,17 @@ class WebDriver(RemoteWebDriver):
     FIREFOX = "FIREFOX"
 
     def __init__(
-            self,
-            load_images=True,
-            user_agent=None,
-            proxy=None,
-            headless=False,
-            driver_type=PHANTOMJS,
-            timeout=16,
-            window_size=(1024, 800),
-            executable_path=None,
-            custom_argument=None,
-            **kwargs
+        self,
+        load_images=True,
+        user_agent=None,
+        proxy=None,
+        headless=False,
+        driver_type=PHANTOMJS,
+        timeout=16,
+        window_size=(1024, 800),
+        executable_path=None,
+        custom_argument=None,
+        **kwargs
     ):
         """
         webdirver 封装，支持chrome、phantomjs 和 firefox
@@ -59,7 +59,7 @@ def __init__(
         self._timeout = timeout
         self._window_size = window_size
         self._executable_path = executable_path
-        self.custom_argument = custom_argument
+        self._custom_argument = custom_argument
 
         self.proxies = {}
         self.user_agent = None
@@ -126,6 +126,11 @@ def firefox_driver(self):
             firefox_options.add_argument("--headless")
             firefox_options.add_argument("--disable-gpu")
 
+        # 添加自定义的配置参数
+        if self._custom_argument:
+            for arg in self._custom_argument:
+                firefox_options.add_argument(arg)
+
         if self._executable_path:
             driver = webdriver.Firefox(
                 capabilities=firefox_capabilities,
@@ -178,9 +183,10 @@ def chrome_driver(self):
             chrome_options.add_argument(
                 "--window-size={},{}".format(self._window_size[0], self._window_size[1])
             )
+
         # 添加自定义的配置参数
-        if self.custom_argument:
-            for arg in self.custom_argument:
+        if self._custom_argument:
+            for arg in self._custom_argument:
                 chrome_options.add_argument(arg)
 
         if self._executable_path:
@@ -221,9 +227,10 @@ def phantomjs_driver(self):
             )
         if not self._load_images:
             service_args.append("--load-images=no")
+
         # 添加自定义的配置参数
-        if self.custom_argument:
-            for arg in self.custom_argument:
+        if self._custom_argument:
+            for arg in self._custom_argument:
                 service_args.append(arg)
 
         if self._executable_path:

Original file line number	Diff line number	Diff line change
`@@ -56,6 +56,7 @@`
`56`	`56`	`window_size=(1024, 800), # 窗口大小`
`57`	`57`	`executable_path=None, # 浏览器路径，默认为默认路径`
`58`	`58`	`render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码`
	`59`	`+ custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数`
`59`	`60`	`)`
`60`	`61`
`61`	`62`	`# 爬虫启动时，重新抓取失败的requests`