fixed 去除多余字符避免SQL执行错误

bajins · bajins · commit 54bf8938217b · 2019-12-30T11:47:03.000+08:00
diff --git a/Pexels.py b/Pexels.py
@@ -9,6 +9,7 @@
 # @Software: PyCharm
 import gc
 import os
+import re
 import threading
 import time
 
@@ -57,11 +58,11 @@ def download_latest_images(page, directory):
 
             info_html = BeautifulSoup(HttpUtil.get("https://www.pexels.com/zh-cn/photo/" + image_id).text,
                                       features="lxml")
-            tags = info_html.find("meta", {"name": "keywords"}).attrs["content"].replace(" ", "").replace("'", "")
+            tags = info_html.find("meta", {"name": "keywords"}).attrs["content"]
             if len(tags) > 0 and tags != "":
                 # 简繁转换
                 tags = zhconv.convert(tags[:len(tags) - 7], 'zh-cn')
-
+                tags = re.sub(r"[^a-z,\u4e00-\u9fa5]+|^,|,$", "", tags).replace(",,", ",")
             s3.execute_commit(f"""
             INSERT OR IGNORE INTO images(image_id,suffix,url,type,page,tags) 
             VALUES('{image_id}','{download_url[download_url.rfind(".") + 1:]}','{download_url}','latest','{page}','{tags}')
@@ -89,6 +90,8 @@ def download_latest_images(page, directory):
                 page = 1
             run_count = 0
 
+    except Exception as e:
+        print(e)
     finally:
         print("当前活跃线程数:", threading.activeCount())
         time.sleep(400)
diff --git a/Wallhaven.py b/Wallhaven.py
@@ -9,7 +9,7 @@
 # @Software: PyCharm
 import gc
 import os
-import platform
+import re
 import threading
 import time
 
@@ -42,7 +42,7 @@ def download_images(url, page, directory):
 
         wait()
 
-        html = BeautifulSoup(HttpUtil.get(url + str(2472)).text, features="lxml")
+        html = BeautifulSoup(HttpUtil.get(url + str(page)).text, features="lxml")
         figure = html.find_all("figure")
         # 获取所有包含指定属性的标签
         page_all = html.find_all(lambda tag: tag.has_attr('original-title'))
@@ -63,6 +63,7 @@ def download_images(url, page, directory):
             tags = ",".join([tag_html.text for tag_html in tags_html]).replace("'", "")
             if len(tags) > 0 and tags != "":
                 tags = TranslationUtil.translate_google(tags).replace("，", ",")
+                tags = re.sub(r"[^a-z,\u4e00-\u9fa5]+|^,|,$", "", tags).replace(",,", ",")
 
             download_url = info_html.find("img", {"id": "wallpaper"}).attrs["src"]
             if len(download_url) <= 0 or download_url == "":
@@ -96,6 +97,8 @@ def download_images(url, page, directory):
                 page = 1
             run_count = 0
 
+    except Exception as e:
+        print(e)
     finally:
         print("当前活跃线程数:", threading.activeCount())
         time.sleep(400)
diff --git a/utils/DatabaseUtil.py b/utils/DatabaseUtil.py
@@ -31,6 +31,9 @@ def select(connect, sql):
         # data = cursor.fetchone()
         # 获取所有数据
         return cursor.fetchall()
+    except sqlite3.OperationalError as e:
+        print(sql)
+        raise e
     finally:
         # 关闭游标
         cursor.close()
@@ -57,6 +60,9 @@ def execute_commit(connect, sql):
         # 操作后获取成功行数
         # return cursor.arraysize
         return cursor.rowcount
+    except sqlite3.OperationalError as e:
+        print(sql)
+        raise e
     finally:
         # 关闭游标
         cursor.close()
