Skip to content

Commit 418379c

Browse files
author
Boris
committed
支持指定浏览器路径
1 parent 7b1e067 commit 418379c

6 files changed

Lines changed: 70 additions & 32 deletions

File tree

‎docs/source_code/Request.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ Request除了支持requests的所有参数外，更需要关心的是框架中
3636
@param random_user_agent: 是否随机User-Agent (True/False) 当setting中的RANDOM_HEADERS设置为True时该参数生效 默认True
3737
@param download_midware: 下载中间件。默认为parser中的download_midware
3838
@param is_abandoned: 当发生异常时是否放弃重试 True/False. 默认False
39+
@param render: 是否用浏览器渲染
3940
--
4041
以下参数于requests参数使用方式一致
4142
@param method: 请求方式，如POST或GET，默认根据data值是否为空来判断

‎docs/source_code/配置文件.md‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,6 @@
99

1010

1111
```python
12-
# redis key
13-
# 任务表模版
14-
TAB_REQUSETS = "{redis_key}:z_requsets"
15-
# 任务失败模板
16-
TAB_FAILED_REQUSETS = "{redis_key}:z_failed_requsets"
17-
# 爬虫状态表模版
18-
TAB_SPIDER_STATUS = "{redis_key}:z_spider_status"
19-
# item 表模版
20-
TAB_ITEM = "{redis_key}:s_{item_name}"
21-
# 爬虫时间记录表
22-
TAB_SPIDER_TIME = "{redis_key}:h_spider_time"
2312

2413
# MYSQL
2514
MYSQL_IP = os.getenv("MYSQL_IP")

‎docs/usage/AirSpider.md‎

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,37 @@ MysqlDB 的具体使用方法见 [MysqlDB](source_code/MysqlDB.md)
192192

193193
RedisDB 的具体使用方法见 [RedisDB](source_code/RedisDB.md)
194194

195-
## 12. 完整的代码示例
196-
197-
[https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py](https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py)
195+
## 12. 浏览器渲染下载
196+
197+
采集动态页面时（Ajax渲染的页面），常用的有两种方案。一种是找接口拼参数，这种方式比较复杂但效率高，需要一定的爬虫功底；另外一种是采用浏览器渲染的方式，直接获取源码，简单方便
198+
199+
使用方式：
200+
```python
201+
def start_requests(self):
202+
yield feapder.Request("https://news.qq.com/", render=True)
203+
```
204+
在返回的Request中传递`render=True`即可
205+
206+
框架支持`CHROME`和`PHANTOMJS`两种浏览器渲染，可通过[配置文件](source_code/配置文件)进行配置。相关配置如下：
207+
208+
```python
209+
# 浏览器渲染
210+
# 浏览器渲染
211+
WEBDRIVER = dict(
212+
pool_size=2, # 浏览器的数量
213+
load_images=False, # 是否加载图片
214+
user_agent=None, # 字符串 或 无参函数，返回值为user_agent
215+
proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
216+
headless=False, # 是否为无头浏览器
217+
driver_type="CHROME", # CHROME 或 PHANTOMJS,
218+
timeout=30, # 请求超时时间
219+
window_size=(1024, 800), # 窗口大小
220+
executable_path=None, # 浏览器路径，默认为默认路径
221+
)
222+
```
223+
224+
## 13. 完整的代码示例
225+
226+
AirSpider：https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py
227+
228+
浏览器渲染：https://github.com/Boris-code/feapder/blob/master/tests/test_rander.py

‎feapder/setting.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,15 @@
4848

4949
# 浏览器渲染
5050
WEBDRIVER = dict(
51-
pool_size=2,
52-
load_images=False,
53-
user_agent=None, # 字符串 或 返回user_agent的函数
54-
proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 返回代理的函数
55-
headless=False,
51+
pool_size=2, # 浏览器的数量
52+
load_images=False, # 是否加载图片
53+
user_agent=None, # 字符串 或 无参函数，返回值为user_agent
54+
proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
55+
headless=False, # 是否为无头浏览器
5656
driver_type="CHROME", # CHROME 或 PHANTOMJS,
57-
timeout=30,
58-
window_size=(1024, 800),
57+
timeout=30, # 请求超时时间
58+
window_size=(1024, 800), # 窗口大小
59+
executable_path=None, # 浏览器路径，默认为默认路径
5960
)
6061

6162
# 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败

‎feapder/templates/project_template/setting.py‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,15 @@
3232
#
3333
# # 浏览器渲染下载
3434
# WEBDRIVER = dict(
35-
# pool_size=2,
36-
# load_images=False,
37-
# user_agent=None, # 字符串 或 返回user_agent的函数
38-
# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 返回代理的函数
39-
# headless=False,
35+
# pool_size=2, # 浏览器的数量
36+
# load_images=False, # 是否加载图片
37+
# user_agent=None, # 字符串 或 无参函数，返回值为user_agent
38+
# proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
39+
# headless=False, # 是否为无头浏览器
4040
# driver_type="CHROME", # CHROME 或 PHANTOMJS,
41-
# timeout=30,
42-
# window_size=(1024, 800),
41+
# timeout=30, # 请求超时时间
42+
# window_size=(1024, 800), # 窗口大小
43+
# executable_path=None, # 浏览器路径，默认为默认路径
4344
# )
4445
#
4546
# # 重新尝试失败的requests 当requests重试次数超过允许的最大重试次数算失败

‎feapder/utils/webdriver.py‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def __init__(
3131
driver_type=PHANTOMJS,
3232
timeout=16,
3333
window_size=(1024, 800),
34+
executable_path=None,
3435
):
3536
"""
3637
@@ -43,6 +44,7 @@ def __init__(
4344
@param proxy xxx.xxx.xxx.xxx:xxxx 或 返回代理的函数
4445
@param timeout: 请求超时时间 默认16s
4546
@param window_size: 屏幕分辨率 (width, height)
47+
@param executable_path: 浏览器路径，默认为默认路径
4648
"""
4749
self._load_images = load_images
4850
self._user_agent = (
@@ -53,6 +55,7 @@ def __init__(
5355
self._headless = headless
5456
self._timeout = timeout
5557
self._window_size = window_size
58+
self._executable_path = executable_path
5659

5760
self.proxies = {}
5861
self.user_agent = None
@@ -120,7 +123,12 @@ def chrome_driver(self):
120123
"--window-size={},{}".format(self._window_size[0], self._window_size[1])
121124
)
122125

123-
driver = webdriver.Chrome(chrome_options=chrome_options)
126+
if self._executable_path:
127+
driver = webdriver.Chrome(
128+
chrome_options=chrome_options, executable_path=self._executable_path
129+
)
130+
else:
131+
driver = webdriver.Chrome(chrome_options=chrome_options)
124132

125133
driver.execute_cdp_cmd(
126134
"Page.addScriptToEvaluateOnNewDocument",
@@ -154,9 +162,16 @@ def phantomjs_driver(self):
154162
if not self._load_images:
155163
service_args.append("--load-images=no")
156164

157-
driver = webdriver.PhantomJS(
158-
service_args=service_args, desired_capabilities=dcap
159-
)
165+
if self._executable_path:
166+
driver = webdriver.PhantomJS(
167+
service_args=service_args,
168+
desired_capabilities=dcap,
169+
executable_path=self._executable_path,
170+
)
171+
else:
172+
driver = webdriver.PhantomJS(
173+
service_args=service_args, desired_capabilities=dcap
174+
)
160175

161176
if self._window_size:
162177
driver.set_window_size(self._window_size[0], self._window_size[1])

0 commit comments

Comments
 (0)