修复浏览器渲染问题

Boris · Boris · commit 5cf33fde32af · 2021-03-19T11:26:25.000+08:00
diff --git a/docs/command/cmdline.md b/docs/command/cmdline.md
@@ -339,7 +339,7 @@ item = SpiderDataItem(**response_data)
 
     可以看出，我们可以直接使用response， response支持xpath表达式
 
-    若想查看都支持哪些函数，可输入`resonse.` 然后敲两次`tab键`，如下：
+    若想查看都支持哪些函数，可输入`response.` 然后敲两次`tab键`，如下：
     ![-w1738](http://markdown-media.oss-cn-beijing.aliyuncs.com/2020/06/21/15927532396490.jpg)
 
 1. 以curl为例，请求百度（通常用来测试post接口比较方便）
diff --git a/docs/usage/AirSpider.md b/docs/usage/AirSpider.md
@@ -221,6 +221,8 @@ WEBDRIVER = dict(
 )
 ```
 
+默认不加载图片，提高渲染速度
+
 ## 13. 完整的代码示例
 
 AirSpider：https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py
diff --git a/feapder/VERSION b/feapder/VERSION
@@ -1 +1 @@
-1.3.2
+1.3.3
diff --git a/feapder/core/spiders/air_spider.py b/feapder/core/spiders/air_spider.py
@@ -82,6 +82,5 @@ def run(self):
                 if Request.webdriver_pool:
                     Request.webdriver_pool.close()
 
-
                 log.info("无任务，爬虫结束")
                 break
diff --git a/feapder/network/request.py b/feapder/network/request.py
@@ -326,7 +326,8 @@ def get_response(self, save_cached=False):
                 )
 
                 response._cached_text = html
-                response.browser = browser
+                # response.browser = browser # 因为浏览器渲染完就释放了，所以不能绑定到response上
+                self._webdriver_pool.put(browser)
             except Exception as e:
                 self._webdriver_pool.remove(browser)
                 raise e
diff --git a/feapder/utils/webdriver.py b/feapder/utils/webdriver.py
@@ -8,7 +8,7 @@
 @email: boris_liu@foxmail.com
 """
 
-import collections
+import queue
 import threading
 
 from selenium import webdriver
@@ -201,27 +201,35 @@ def __getattr__(self, name):
 @Singleton
 class WebDriverPool:
     def __init__(self, pool_size=5, **kwargs):
-        self.queue = collections.deque(maxlen=pool_size)
+        self.queue = queue.Queue(maxsize=pool_size)
         self.kwargs = kwargs
         self.lock = threading.RLock()
+        self.driver_count = 0
+
+    @property
+    def is_full(self):
+        return self.driver_count >= self.queue.maxsize
 
     def get(self):
-        if not len(self.queue) >= self.queue.maxlen:
+        if not self.is_full:
             with self.lock:
-                if not len(self.queue) >= self.queue.maxlen:
+                if not self.is_full:
                     driver = WebDriver(**self.kwargs)
-                    self.queue.append(driver)
-
-        driver = self.queue.popleft()
-        self.queue.append(driver)
+                    self.queue.put(driver)
+                    self.driver_count += 1
 
+        driver = self.queue.get()
         return driver
 
+    def put(self, driver):
+        self.queue.put(driver)
+
     def remove(self, driver):
         driver.quit()
-        self.queue.remove(driver)
+        self.driver_count -= 1
 
     def close(self):
-        while self.queue:
-            driver = self.queue.pop()
+        while not self.queue.empty():
+            driver = self.queue.get()
             driver.quit()
+            self.driver_count -= 1
diff --git a/tests/test_rander.py b/tests/test_rander.py
@@ -4,16 +4,16 @@
 class XueQiuSpider(feapder.AirSpider):
     def start_requests(self):
         for i in range(10):
-            yield feapder.Request("https://news.qq.com/", render=True)
+            yield feapder.Request("https://news.qq.com/#{}".format(i), render=True)
 
     def parse(self, request, response):
-        print(response.browser)
-        print(response)
+        print(response.cookies.get_dict())
+        print("response.url ", response.url)
 
-        article_list = response.xpath('//div[@class="detail"]')
-        for article in article_list:
-            title = article.xpath("string(.//a)").extract_first()
-            print(title)
+        # article_list = response.xpath('//div[@class="detail"]')
+        # for article in article_list:
+        #     title = article.xpath("string(.//a)").extract_first()
+        #     print(title)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_rander2.py b/tests/test_rander2.py
@@ -11,8 +11,8 @@ def start_requests(self):
             yield feapder.Request("https://news.qq.com/#{}".format(i), render=True)
 
     def parse(self, request, response):
-        print(response.browser)
-        print(response)
+        print(response.cookies.get_dict())
+        print("response.url ", response.url)
 
         article_list = response.xpath('//div[@class="detail"]')
         for article in article_list:
@@ -21,4 +21,6 @@ def parse(self, request, response):
 
 
 if __name__ == "__main__":
-    XueQiuSpider(thread_count=10, redis_key="feapter:test_rander").start()
+    XueQiuSpider(
+        thread_count=10, redis_key="feapter:test_rander", delete_keys=True
+    ).start()

Original file line number	Diff line number	Diff line change
`@@ -221,6 +221,8 @@ WEBDRIVER = dict(`
`221`	`221`	`)`
`222`	`222`	```
`223`	`223`
	`224`	`+默认不加载图片，提高渲染速度`
	`225`	`+`
`224`	`226`	`## 13. 完整的代码示例`
`225`	`227`
`226`	`228`	`AirSpider：https://github.com/Boris-code/feapder/blob/master/tests/air-spider/test_air_spider.py`
Original file line number	Diff line number	Diff line change
`@@ -326,7 +326,8 @@ def get_response(self, save_cached=False):`
`326`	`326`	`)`
`327`	`327`
`328`	`328`	`response._cached_text = html`
`329`		`- response.browser = browser`
	`329`	`+ # response.browser = browser # 因为浏览器渲染完就释放了，所以不能绑定到response上`
	`330`	`+ self._webdriver_pool.put(browser)`
`330`	`331`	`except Exception as e:`
`331`	`332`	`self._webdriver_pool.remove(browser)`
`332`	`333`	`raise e`