Skip to content

Commit 216e68d

Browse files
Boris-codeBoris-code
committed
添加浏览器拦截数据文档
1 parent a7ea197 commit 216e68d

2 files changed

Lines changed: 161 additions & 1 deletion

File tree

‎docs/source_code/浏览器渲染.md‎

Lines changed: 109 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ WEBDRIVER = dict(
3030
executable_path=None, # 浏览器路径，默认为默认路径
3131
render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码
3232
custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数
33+
xhr_url_regexes=None, # 拦截xhr接口，支持正则，数组类型
3334
)
3435
```
3536

@@ -138,14 +139,121 @@ class TestRender(feapder.AirSpider):
138139

139140
# response也是可以正常使用的
140141
# response.xpath("//title")
141-
# response.text
142+
143+
# 若有滚动，可通过如下方式更新response，使其加载滚动后的内容
144+
# response.text = browser.page_source
142145

143146

144147
if __name__ == "__main__":
145148
TestRender().start()
146149

147150
```
148151

152+
## 拦截xhr数据
153+
154+
### 设置拦截规则
155+
156+
```python
157+
WEBDRIVER = dict(
158+
...
159+
xhr_url_regexes=[
160+
"接口1正则",
161+
"接口2正则",
162+
]
163+
)
164+
```
165+
166+
### 获取数据
167+
168+
```python
169+
browser: WebDriver = response.browser
170+
text = browser.xhr_text("接口1正则")
171+
```
172+
173+
### 获取json格式数据
174+
175+
```python
176+
browser: WebDriver = response.browser
177+
data = browser.xhr_json("接口1正则")
178+
```
179+
180+
### 获取response
181+
182+
```python
183+
browser: WebDriver = response.browser
184+
xhr_response = browser.xhr_response("接口1正则")
185+
print("请求接口", xhr_response.request.url)
186+
print("请求头", xhr_response.request.headers)
187+
print("请求体", xhr_response.request.data)
188+
print("返回头", xhr_response.headers)
189+
print("返回地址", xhr_response.url)
190+
print("返回内容", xhr_response.content)
191+
```
192+
193+
194+
### 示例：
195+
196+
需求：
197+
![](http://markdown-media.oss-cn-beijing.aliyuncs.com/2021/12/30/16408610725756.jpg)
198+
199+
代码：
200+
```python
201+
import time
202+
203+
import feapder
204+
from feapder.utils.webdriver import WebDriver
205+
206+
207+
class TestRender(feapder.AirSpider):
208+
__custom_setting__ = dict(
209+
WEBDRIVER=dict(
210+
pool_size=1, # 浏览器的数量
211+
load_images=True, # 是否加载图片
212+
user_agent=None, # 字符串 或 无参函数，返回值为user_agent
213+
proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
214+
headless=False, # 是否为无头浏览器
215+
driver_type="CHROME", # CHROME、PHANTOMJS、FIREFOX
216+
timeout=30, # 请求超时时间
217+
window_size=(1024, 800), # 窗口大小
218+
executable_path=None, # 浏览器路径，默认为默认路径
219+
render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码
220+
custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数
221+
xhr_url_regexes=[
222+
"/ad",
223+
], # 拦截 http://www.spidertools.cn/spidertools/ad 接口
224+
)
225+
)
226+
227+
def start_requests(self):
228+
yield feapder.Request("http://www.spidertools.cn", render=True)
229+
230+
def parse(self, request, response):
231+
browser: WebDriver = response.browser
232+
time.sleep(3)
233+
234+
# 获取接口数据 文本类型
235+
ad = browser.xhr_text("/ad")
236+
print(ad)
237+
238+
# 获取接口数据 转成json，本例因为返回的接口是文本，所以不转了
239+
# browser.xhr_json("/ad")
240+
241+
xhr_response = browser.xhr_response("/ad")
242+
print("请求接口", xhr_response.request.url)
243+
# 请求头目前获取的不完整
244+
print("请求头", xhr_response.request.headers)
245+
print("请求体", xhr_response.request.data)
246+
print("返回头", xhr_response.headers)
247+
print("返回地址", xhr_response.url)
248+
print("返回内容", xhr_response.content)
249+
250+
251+
if __name__ == "__main__":
252+
TestRender().start()
253+
254+
```
255+
256+
149257
## 关闭当前浏览器
150258

151259
```python

‎tests/test_rander_xhr.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import time
2+
3+
import feapder
4+
from feapder.utils.webdriver import WebDriver
5+
6+
7+
class TestRender(feapder.AirSpider):
8+
__custom_setting__ = dict(
9+
WEBDRIVER=dict(
10+
pool_size=1, # 浏览器的数量
11+
load_images=True, # 是否加载图片
12+
user_agent=None, # 字符串 或 无参函数，返回值为user_agent
13+
proxy=None, # xxx.xxx.xxx.xxx:xxxx 或 无参函数，返回值为代理地址
14+
headless=False, # 是否为无头浏览器
15+
driver_type="CHROME", # CHROME、PHANTOMJS、FIREFOX
16+
timeout=30, # 请求超时时间
17+
window_size=(1024, 800), # 窗口大小
18+
executable_path=None, # 浏览器路径，默认为默认路径
19+
render_time=0, # 渲染时长，即打开网页等待指定时间后再获取源码
20+
custom_argument=["--ignore-certificate-errors"], # 自定义浏览器渲染参数
21+
xhr_url_regexes=[
22+
"/ad",
23+
], # 拦截 http://www.spidertools.cn/spidertools/ad 接口
24+
)
25+
)
26+
27+
def start_requests(self):
28+
yield feapder.Request("http://www.spidertools.cn", render=True)
29+
30+
def parse(self, request, response):
31+
browser: WebDriver = response.browser
32+
time.sleep(3)
33+
34+
# 获取接口数据 文本类型
35+
ad = browser.xhr_text("/ad")
36+
print(ad)
37+
38+
# 获取接口数据 转成json，本例因为返回的接口是文本，所以不转了
39+
# browser.xhr_json("/ad")
40+
41+
xhr_response = browser.xhr_response("/ad")
42+
print("请求接口", xhr_response.request.url)
43+
# 请求头目前获取的不完整
44+
print("请求头", xhr_response.request.headers)
45+
print("请求体", xhr_response.request.data)
46+
print("返回头", xhr_response.headers)
47+
print("返回地址", xhr_response.url)
48+
print("返回内容", xhr_response.content)
49+
50+
51+
if __name__ == "__main__":
52+
TestRender().start()

0 commit comments

Comments
 (0)