-
Notifications
You must be signed in to change notification settings - Fork 546
Expand file tree
/
Copy pathproxy_pool.py
More file actions
69 lines (57 loc) · 1.99 KB
/
Copy pathproxy_pool.py
File metadata and controls
69 lines (57 loc) · 1.99 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# -*- coding: utf-8 -*-
"""
Created on 2022/10/19 10:40 AM
---------
@summary:
---------
@author: Boris
@email: boris_liu@foxmail.com
"""
from queue import Queue
import requests
import feapder.setting as setting
from feapder.network.proxy_pool.base import BaseProxyPool
from feapder.utils import metrics
from feapder.utils import tools
class ProxyPool(BaseProxyPool):
"""
通过API提取代理，存储在内存中，无代理时会自动提取
API返回的代理以 \r\n 分隔
"""
def __init__(self, proxy_api=None, **kwargs):
self.proxy_api = proxy_api or setting.PROXY_EXTRACT_API
self.proxy_queue = Queue()
def format_proxy(self, proxy):
return {"http": "http://" + proxy, "https": "http://" + proxy}
@tools.retry(3, interval=5)
def pull_proxies(self):
resp = requests.get(self.proxy_api)
proxies = resp.text.strip()
resp.close()
if "{" in proxies or not proxies:
raise Exception("获取代理失败", proxies)
# 使用 /r/n 分隔
return proxies.split("\r\n")
def get_proxy(self):
try:
if self.proxy_queue.empty():
proxies = self.pull_proxies()
for proxy in proxies:
self.proxy_queue.put_nowait(proxy)
metrics.emit_counter("total", 1, classify="proxy")
proxy = self.proxy_queue.get_nowait()
self.proxy_queue.put_nowait(proxy)
metrics.emit_counter("used_times", 1, classify="proxy")
return self.format_proxy(proxy)
except Exception as e:
tools.send_msg("获取代理失败", level="error")
raise Exception("获取代理失败", e)
def del_proxy(self, proxy):
"""
@summary: 删除代理
---------
@param proxy: ip:port
"""
if proxy in self.proxy_queue.queue:
self.proxy_queue.queue.remove(proxy)
metrics.emit_counter("invalid", 1, classify="proxy")