update 添加异步

bajins · bajins · commit eb343e250e2d · 2019-12-31T10:21:15.000+08:00
diff --git a/Pexels.py b/Pexels.py
@@ -30,11 +30,7 @@ def download_latest_images(page, directory):
     try:
         dir_size = FileUtil.count_dir_size(directory)
         if dir_size >= 1073741824:
-            print(FileUtil.size_unit_format(dir_size))
-            # raise IOError("存储的图片超过1GB")
-            print(os.system("rclone move /home/reptile-python/images/ gdrive:/images --min-size 100k"))
-            print(FileUtil.size_unit_format(FileUtil.count_dir_size(directory)))
-
+            asyncio.run(move(directory, dir_size))
         wait()
 
         html = BeautifulSoup(HttpUtil.get("https://www.pexels.com/zh-cn/new-photos?page=" + str(page)).text,
@@ -76,7 +72,6 @@ def download_latest_images(page, directory):
                 # 每张图片启用单个线程下载
                 # done = ThreadPool.pool.submit(HttpUtil.download_file, download_url, directory, image_name)
                 # done.add_done_callback(ThreadPool.thread_call_back)
-                # threading.Timer(30, HttpUtil.download_file, (download_url, directory, image_name))
                 asyncio.run(HttpUtil.download_one_async(download_url, directory, image_name))
 
         global run_count
@@ -96,9 +91,14 @@ def download_latest_images(page, directory):
         print(e)
     finally:
         print("当前活跃线程数:", threading.activeCount())
-        if psutil.virtual_memory().percent < 80:
-            time.sleep(400)
-            download_latest_images(page, directory)
+        download_latest_images(page, directory)
+
+
+async def move(directory, dir_size):
+    print(FileUtil.size_unit_format(dir_size))
+    # raise IOError("存储的图片超过1GB")
+    print(os.system("rclone move /home/reptile-python/images/ gdrive:/images --min-size 100k"))
+    print(FileUtil.size_unit_format(FileUtil.count_dir_size(directory)))
 
 
 def wait():
diff --git a/Wallhaven.py b/Wallhaven.py
@@ -7,6 +7,7 @@
 # @Project: tool-gui-python
 # @Package: 
 # @Software: PyCharm
+import asyncio
 import gc
 import os
 import re
@@ -35,10 +36,7 @@ def download_images(url, page, directory):
     try:
         dir_size = FileUtil.count_dir_size(directory)
         if dir_size >= 1073741824:
-            print(FileUtil.size_unit_format(dir_size))
-            # raise IOError("存储的图片超过1GB")
-            print(os.system("rclone move /home/reptile-python/images/ onedrive:/images --min-size 100k"))
-            print(FileUtil.size_unit_format(FileUtil.count_dir_size(directory)))
+            asyncio.run(move(directory, dir_size))
 
         wait()
 
@@ -80,9 +78,9 @@ def download_images(url, page, directory):
             # if not os.path.exists(name):
             if not os.path.isfile(os.path.join(directory, image_name)):
                 # 每张图片启用单个线程下载
-                done = ThreadPool.pool.submit(HttpUtil.download_file, download_url, directory, image_name)
+                # done = ThreadPool.pool.submit(HttpUtil.download_file, download_url, directory, image_name)
                 # done.add_done_callback(ThreadPool.thread_call_back)
-                # threading.Timer(30, HttpUtil.download_file, (download_url, directory, image_name))
+                asyncio.run(HttpUtil.download_one_async(download_url, directory, image_name))
 
         global run_count
         run_count += 1
@@ -148,6 +146,13 @@ def get_tag(page):
         get_tag(page + 1)
 
 
+async def move(directory, dir_size):
+    print(FileUtil.size_unit_format(dir_size))
+    # raise IOError("存储的图片超过1GB")
+    print(os.system("rclone move /home/reptile-python/images/ gdrive:/images --min-size 100k"))
+    print(FileUtil.size_unit_format(FileUtil.count_dir_size(directory)))
+
+
 def wait():
     """
     垃圾回收
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,7 @@
+aiofiles==0.4.0
+aiohttp==3.6.2
+async-timeout==3.0.1
+attrs==19.3.0
 beautifulsoup4==4.8.2
 botocore==1.13.19
 certifi==2019.11.28
@@ -7,6 +11,7 @@ googletrans==2.4.0
 idna==2.8
 jmespath==0.9.4
 lxml==4.4.2
+multidict==4.7.2
 numpy==1.18.0
 pandas==0.25.3
 Pillow==6.2.1
@@ -20,4 +25,5 @@ selenium==3.141.0
 six==1.13.0
 soupsieve==1.9.5
 urllib3==1.25.7
+yarl==1.4.2
 zhconv==1.4.0
diff --git a/utils/HttpUtil.py b/utils/HttpUtil.py
@@ -20,8 +20,6 @@
 import requests
 import urllib3
 
-from utils import ThreadPool
-
 USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " \
              "Chrome/77.0.3865.75 Safari/537.36 "
 
