2020import requests
2121import urllib3
2222
23- USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " \
24- "Chrome/77.0.3865.75 Safari/537.36 "
2523
2624# 去除警告
25+ import constants
26+
2727requests .packages .urllib3 .disable_warnings ()
2828# 如果请求失败默认重试次数
2929requests .adapters .DEFAULT_RETRIES = 5
3030
3131
32+ def get_session ():
33+ """
34+ 获取session
35+ :param url:请求地址
36+ :param data:数据,map或dict格式
37+ :return:
38+ """
39+ session = requests .sessions .Session ()
40+ # 关闭多余的连接
41+ session .keep_alive = False
42+ session .headers ["User-Agent" ] = constants .USER_AGENT
43+ session .verify = False
44+ session .timeout = 600
45+ return session
46+
47+
3248def get (url , data = None ):
3349 """
3450 get请求
@@ -39,7 +55,7 @@ def get(url, data=None):
3955 session = requests .sessions .Session ()
4056 # 关闭多余的连接
4157 session .keep_alive = False
42- return session .get (url , params = data , headers = {"User-Agent" : USER_AGENT }, verify = False , timeout = 600 )
58+ return session .get (url , params = data , headers = {"User-Agent" : constants . USER_AGENT }, verify = False , timeout = 600 )
4359
4460
4561def post (url , data ):
@@ -49,7 +65,7 @@ def post(url, data):
4965 :param data:数据,map或dict格式
5066 :return:
5167 """
52- return requests .post (url , data , headers = {"User-Agent" : USER_AGENT }, verify = False , timeout = 600 )
68+ return requests .post (url , data , headers = {"User-Agent" : constants . USER_AGENT }, verify = False , timeout = 600 )
5369
5470
5571def delete (url , data ):
@@ -59,7 +75,7 @@ def delete(url, data):
5975 :param data:数据,map或dict格式
6076 :return:
6177 """
62- return requests .delete (url = url , params = data , headers = {"User-Agent" : USER_AGENT }, verify = False , timeout = 600 )
78+ return requests .delete (url = url , params = data , headers = {"User-Agent" : constants . USER_AGENT }, verify = False , timeout = 600 )
6379
6480
6581def get_json (url , data ):
@@ -101,7 +117,7 @@ def download_big_file_urlib(url, mkdir, name=""):
101117
102118 req = Request (url )
103119 # 增加header头信息
104- req .add_header ('User-Agent' , USER_AGENT )
120+ req .add_header ('User-Agent' , constants . USER_AGENT )
105121
106122 response = urlopen (req )
107123 while True :
@@ -143,7 +159,7 @@ def download_big_file(url, mkdir, name=""):
143159 name = os .path .join (mkdir , name )
144160
145161 start_time = time .time ()
146- req = requests .get (url , stream = True , headers = {"User-Agent" : USER_AGENT }, verify = False )
162+ req = requests .get (url , stream = True , headers = {"User-Agent" : constants . USER_AGENT }, verify = False )
147163 with req as r :
148164 content_length = int (r .headers ['content-length' ])
149165 print (name , 'content-length: %dB/%.2fKB/%.2fMB' % (
@@ -190,7 +206,7 @@ def download_file(url, mkdir, name=""):
190206 # 判断文件是否存在
191207 # if not os.path.exists(name):
192208 if not os .path .isfile (name ):
193- with requests .get (url , headers = {"User-Agent" : USER_AGENT }, verify = False , timeout = 600 ) as req :
209+ with requests .get (url , headers = {"User-Agent" : constants . USER_AGENT }, verify = False , timeout = 600 ) as req :
194210 with open (name , "wb" ) as f :
195211 f .write (req .content )
196212 return name
0 commit comments