优化及添加cookie池文档

Boris-code · Boris-code · commit eb2d5d520752 · 2021-12-02T09:59:42.000+08:00
diff --git a/docs/source_code/CookiePool.md b/docs/source_code/CookiePool.md
@@ -0,0 +1,35 @@
+# Cookie池使用说明
+
+cookie池分为三种，使用场景如下
+1. `PageCookiePool`：从不需要登录的页面获取cookie
+2. `LoginCookiePool`：有大量账号，自动切换登录，获取登录后cookie
+3. `LimitTimesUserPool`：账号昂贵，需要限制使用次数、使用频率、使用时间段、账号独占使用
+
+## PageCookiePool
+
+### 快速尝鲜
+```
+from feapder.network.cookie_pool import PageCookiePool
+
+cookie_pool = PageCookiePool(redis_key="test:page_cookie_pool", page_url="https://www.baidu.com")
+cookies = cookie_pool.get_cookie()
+print(cookies)
+```
+
+执行上面代码，会自动获取百度的cookie，存放到redis里，然后返回获取到的cookie
+
+### 使用进阶
+
+PageCookiePool参数
+
+- redis_key: 
+- page_url: 生产cookie的url
+- min_cookies: 最小cookie数
+- must_contained_keys: cookie 必须包含的key
+- keep_alive: 当cookie数量足够是是否保持随时待命，生产cookie的状态。False为否，满足则退出
+- **kwargs：WebDriver的一些参数，具体可参考源码注释
+
+
+
+## LoginCookiePool
+## LimitTimesUserPool
diff --git a/feapder/network/cookie_pool.py b/feapder/network/cookie_pool.py
@@ -410,16 +410,14 @@ class LimitTimesUserStatus(Enum):
 class LimitTimesUser:
     """
     有次数限制的账户
-    基于本地做的缓存，不支持多进程调用
     """
 
-    ACCOUNT_INFO_KEY = "accounts:h_account_info"  # 存储cookie的redis key
-    SITE_NAME = ""  # 网站名
-
     redisdb = None
 
     def __init__(
         self,
+        redis_key,
+        *,
         username,
         password,
         max_search_times,
@@ -428,13 +426,15 @@ def __init__(
         **kwargs,
     ):
         """
+        @param redis_key: cookie存储位置
         @param username:
         @param password:
         @param max_search_times:
         @param proxies:
         @param search_interval: 调用时间间隔。 支持元组 指定间隔的时间范围 如（5，10）即5到10秒；或直接传整数
         """
         self.__dict__.update(kwargs)
+        self.redis_key = "{}:h_cookie_pool".format(redis_key)
         self.username = username
         self.password = password
         self.max_search_times = max_search_times
@@ -487,7 +487,7 @@ def __eq__(self, other):
         return self.username == other.username
 
     def sync_account_info_from_redis(self):
-        account_info = self.redisdb.hget(self.ACCOUNT_INFO_KEY, self.username)
+        account_info = self.redisdb.hget(self.redis_key, self.username)
         if account_info:
             account_info = eval(account_info)
             self.account_info.update(account_info)
@@ -499,15 +499,11 @@ def cookies(self):
 
     def set_cookies(self, cookies):
         self.account_info["cookies"] = cookies
-        return self.redisdb.hset(
-            self.ACCOUNT_INFO_KEY, self.username, self.account_info
-        )
+        return self.redisdb.hset(self.redis_key, self.username, self.account_info)
 
     def set_login_time(self, login_time=None):
         self.account_info["login_time"] = login_time or time.time()
-        return self.redisdb.hset(
-            self.ACCOUNT_INFO_KEY, self.username, self.account_info
-        )
+        return self.redisdb.hset(self.redis_key, self.username, self.account_info)
 
     def get_login_time(self):
         return self.account_info.get("login_time")
@@ -549,9 +545,7 @@ def update_status(self):
         self.account_info["search_times"] += 1
         self.account_info["last_search_time"] = time.time()
 
-        return self.redisdb.hset(
-            self.ACCOUNT_INFO_KEY, self.username, self.account_info
-        )
+        return self.redisdb.hset(self.redis_key, self.username, self.account_info)
 
     @property
     def search_times(self):
@@ -563,7 +557,7 @@ def search_times(self):
             self.account_info["search_times"] = 0
             self.account_info["init_search_times_time"] = current_time
 
-            self.redisdb.hset(self.ACCOUNT_INFO_KEY, self.username, self.account_info)
+            self.redisdb.hset(self.redis_key, self.username, self.account_info)
 
         return self.account_info["search_times"]
 
@@ -583,9 +577,7 @@ def is_at_work_time(self):
 
     def del_cookie(self):
         self.account_info["cookies"] = {}
-        return self.redisdb.hset(
-            self.ACCOUNT_INFO_KEY, self.username, self.account_info
-        )
+        return self.redisdb.hset(self.redis_key, self.username, self.account_info)
 
     def create_cookie(self):
         """
@@ -624,9 +616,9 @@ def login(self):
         except Exception as e:
             log.exception(e)
             send_msg(
-                msg=f"{self.SITE_NAME} {self.username} 账号登陆异常 exception: {str(e)}",
+                msg=f"{self.redis_key} {self.username} 账号登陆异常 exception: {str(e)}",
                 level="error",
-                message_prefix=f"{self.SITE_NAME} {self.username} 账号登陆异常",
+                message_prefix=f"{self.redis_key} {self.username} 账号登陆异常",
             )
 
         log.info("登录失败 {}".format(self.username))
@@ -636,14 +628,21 @@ def login(self):
 
 class LimitTimesUserPool:
     """
-    限制查询次数的用户的User pool
-    基于本地做的缓存，不支持多进程调用
+    限制查询次数的用户的UserPool
     """
 
     LOAD_USER_INTERVAL = 60
 
-    def __init__(self, *, accounts_dict, limit_user_class, support_more_client=True):
+    def __init__(
+        self,
+        redis_key,
+        *,
+        accounts_dict: dict,
+        limit_user_class: LimitTimesUser.__class__,
+        support_more_client: bool = True,
+    ):
         """
+        @param redis_key: cookie存储位置
         @param accounts_dic: 账户信息字典
             {
                 "15011300228": {
@@ -657,6 +656,7 @@ def __init__(self, *, accounts_dict, limit_user_class, support_more_client=True)
         @param limit_user_class: 用户重写的 limit_user_class
         @param support_more_client: 是否支持多客户端 即多线程 多进程模式 (可能在计数上及使用频率上有些误差)
         """
+        self.redis_key = redis_key
         self.accounts_dict = accounts_dict
         self.limit_user_class = limit_user_class
 
@@ -675,7 +675,9 @@ def __load_users(self, username=None):
             if username and username != _username:
                 continue
 
-            limit_times_users = self.limit_user_class(username=_username, **detail)
+            limit_times_users = self.limit_user_class(
+                self.redis_key, username=_username, **detail
+            )
             if limit_times_users in self.limit_times_users:
                 continue
 
diff --git a/feapder/utils/webdriver.py b/feapder/utils/webdriver.py
@@ -193,10 +193,10 @@ def chrome_driver(self):
 
         if self._executable_path:
             driver = webdriver.Chrome(
-                chrome_options=chrome_options, executable_path=self._executable_path
+                options=chrome_options, executable_path=self._executable_path
             )
         else:
-            driver = webdriver.Chrome(chrome_options=chrome_options)
+            driver = webdriver.Chrome(options=chrome_options)
 
         # 隐藏浏览器特征
         with open(os.path.join(os.path.dirname(__file__), "./js/stealth.min.js")) as f:
diff --git a/tests/test_cookie_pool.py b/tests/test_cookie_pool.py
@@ -8,8 +8,23 @@
 @email: boris_liu@foxmail.com
 """
 
-from feapder.network.cookie_pool import LoginCookiePool
+from feapder.network.cookie_pool import PageCookiePool,LoginCookiePool, LimitTimesUserPool
 
-cookie_pool = LoginCookiePool("test:login_cookie_pool", tab_userbase="test_userbase")
-cookie = cookie_pool.get_cookie()
-print(cookie)
+
+import unittest
+
+
+
+class TestCookiePool(unittest.TestCase):
+    def test_page_cookie_pool(self):
+        cookie_pool = PageCookiePool(redis_key="test:page_cookie_pool", page_url="https://www.baidu.com")
+        cookies = cookie_pool.get_cookie()
+        print(cookies)
+
+    def test_login_cookie_pool(self):
+        cookie_pool = LoginCookiePool("test:login_cookie_pool", tab_userbase="test_userbase")
+        cookie = cookie_pool.get_cookie()
+        print(cookie)
+
+if __name__ == "__main__":
+    unittest.main()
