diff --git a/.gitignore b/.gitignore index 62c8935..0b71a72 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea/ \ No newline at end of file +.idea/ +spider/jianshu/start.sh \ No newline at end of file diff --git a/auto/qmm.py b/auto/qmm.py index 5dd14b4..7591ca2 100644 --- a/auto/qmm.py +++ b/auto/qmm.py @@ -172,5 +172,5 @@ def _finance_sign(self, driver): if __name__ == '__main__': - qmm = QMM(sleep=3, months='2-3', days='1-31') + qmm = QMM(sleep=3, months='4', days='1-30') qmm.start() diff --git a/blogs/read.py b/blogs/read.py index 7d9b598..1df643a 100644 --- a/blogs/read.py +++ b/blogs/read.py @@ -1,5 +1,6 @@ -import math import time + +import math import numpy import requests from bs4 import BeautifulSoup @@ -54,7 +55,7 @@ def count(self): bs = BeautifulSoup(resp.text, 'html.parser') avatar = bs.find(class_='avatar') # 头像 - self.avatar = 'https:' + avatar.img['src'] + self.avatar = avatar.img['src'] nickname = bs.find(class_='name') # 昵称 self.nickname = nickname.text diff --git a/logger/log.py b/logger/log.py index ae9d05e..4422c59 100644 --- a/logger/log.py +++ b/logger/log.py @@ -5,21 +5,20 @@ import logging.config -class Logger(object): - def __init__(self): +class Logger: + def __init__(self, log_name='info.log'): """日志相关配置""" path = 'logs' if not os.path.exists(path): os.mkdir(path) - path = 'logging.yml' + path = os.path.dirname(__file__) + '/logging.yml' if os.path.exists(path): with open(path, 'r', encoding='utf-8') as f: - config = yaml.load(f) + config = yaml.safe_load(f) logging.config.dictConfig(config) + logging.basicConfig(filename=log_name) else: - logging.basicConfig(level='INFO', filename='info.log', + logging.basicConfig(level='INFO', filename=log_name, format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') - - logging.info(">>> Load logger config") diff --git a/orm/sqlalchemy_crud.py b/orm/sqlalchemy_crud.py index 8238d6b..50bd9e1 100644 --- a/orm/sqlalchemy_crud.py +++ b/orm/sqlalchemy_crud.py @@ -25,7 +25,7 @@ def __str__(self) -> str: # 创建数据库连接（'数据库类型+数据库驱动名称://用户名:密码@ip地址:端口/数据库名'） -conn = "postgresql+psycopg2://postgres:@:5432/postgres" +conn = "postgresql+psycopg2://postgres:@:5432/postgres" engine = create_engine(conn, encoding='UTF-8', echo=False) # echo=True表示输出执行日志,默认为False # 删除映射数据表(如果存在) diff --git a/scrapys/together/together/spiders/yiqi.py b/scrapys/together/together/spiders/yiqi.py index 0ad3e69..89432ce 100644 --- a/scrapys/together/together/spiders/yiqi.py +++ b/scrapys/together/together/spiders/yiqi.py @@ -14,13 +14,12 @@ def __init__(self, name=None, **kwargs): super().__init__(name, **kwargs) self.getUserById = 'http://api.wondertech.com.cn/user/v2/users/getById' # self.likeUser = 'http://api.wondertech.com.cn/fmvoice/v1/voice/vlike' - self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiwiZXhw' \ - 'IjoxNTQ4OTAxMDM4LCJ1c2VySWQiOjU4NjQzfQ.-N6EqLyAU_NvXqcNTT' \ - 'awkpfY_6lidC_XjvYJAnHr-AfBTCz2kT1-Sht_sTDZZOB_lDX6bruHTTJ8SzfZyX1JkA' + self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiwiZXhwIjoxNTU3MTIxMjQxLCJ1c2VySWQiOjU4NjQzfQ.lLvZd0L-ged9WdbQmj9ATS10l_GcyPLIR6P2-THjeZ9VA-iifxpfGBQAFtHPctN0aDHcyFWixiWVb5dzUn7JHw' + self.formData = {'token': self.token} def start_requests(self): - for user_id in range(307478, 320000): + for user_id in range(309130, 350000): self.formData['id'] = str(user_id) yield FormRequest(self.getUserById, formdata=self.formData, callback=self.parse, meta={'uid': user_id}) diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py new file mode 100644 index 0000000..4b968eb --- /dev/null +++ b/spider/bihu/robot.py @@ -0,0 +1,126 @@ +import random +import sys +import time + +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +argv = sys.argv +argv_length = len(argv) + + +def sleep(min_seconds=3, max_seconds=10): + """随机休眠""" + time.sleep(random.randint(min_seconds, max_seconds)) + + +# 无头模式 +options = Options() +options.add_argument('--headless') +options.add_argument('log-level=3') +options.add_argument('--no-sandbox') +options.add_argument('--disable-gpu') +options.add_argument('window-size=1366x728') +options.add_argument('--disable-dev-shm-usage') +driver = webdriver.Chrome(chrome_options=options) + +# 登录币乎 +bihu = "https://bihu.com" +bihu_login = "{}/login".format(bihu) +driver.get(bihu_login) + +driver.maximize_window() + +username = argv[1] if argv_length > 1 else input("请输入用户名\n") +driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[1]/input').send_keys( + username) +password = argv[2] if argv_length > 2 else input("请输入登录密码\n") +driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[2]/input').send_keys( + password) + +driver.find_element_by_class_name("LoaderButton").click() +sleep() + +# 登录成功 +print(driver.current_url) +if driver.current_url == bihu + "/": + + # 1. 点赞长文 + bs = BeautifulSoup(driver.page_source, 'html.parser') + item_list = bs.find_all("li", class_="ArticleItem") + articles = {} + for article_item in item_list: + # 解析ArticleItem + user_info = article_item.find("div", class_="user-info") + p_list = user_info.find_all("p") + publish_time = p_list[-1].text + + bottom = article_item.find("div", class_="item-bottom") + bottom_list = bottom.find_all("span") + # Key数 && 点赞数 && 评论数 + key, like, comment = bottom_list + key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + + content_info = article_item.find("div", class_="content-info") + info = content_info.find("a") + articles[info["href"]] = int(like_num) + + num = 0 + # 点赞操作 + for key, value in sorted(articles.items(), key=lambda item: item[1]): + if num >= 6: break + + article = "{}{}".format(bihu, key) + driver.get(article) + sleep() + target = driver.find_element_by_class_name("collect-div") + driver.execute_script("arguments[0].scrollIntoView()", target) + sleep() + + details = driver.find_element_by_class_name("article-details-center2") + buttons = details.find_elements_by_tag_name("button") + if len(buttons) == 3: + num += 1 + buttons[1].click() + sleep() + + print("长文已点赞完...") + + # 2. 点赞微文 + driver.get("https://bihu.com/shortcontents") + sleep() + + i = 0 + while True: + short_items = driver.find_elements_by_class_name("ShortItem") + short_items_length = len(short_items) + for index, short_item in enumerate(short_items): + bottom = short_item.find_element_by_class_name("item-bottom") + buttons = bottom.find_elements_by_tag_name("button") + like_button = buttons[1] + if like_button and like_button.is_enabled(): + like_button.click() + sleep(1, 3) + i += 1 + if index + 1 == short_items_length: + sleep(5, 10) + if i >= 100: break + + # bs = BeautifulSoup(driver.page_source, 'html.parser') + # item_list = bs.find_all("li", class_="ShortItem") + # num = 0 + # for article_item in item_list: + # if num >= 100: break + # # 解析ShortItem + # user_info = article_item.find("div", class_="user-info") + # p_list = user_info.find_all("p") + # publish_time = p_list[-1].text + # + # bottom = article_item.find("div", class_="item-bottom") + # bottom_list = bottom.find_all("span") + # # Key数 && 点赞数 && 评论数 + # key, like, comment = bottom_list + # key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + + print("微文已点赞完...") diff --git a/wechat/tip.py b/wechat/tip.py new file mode 100644 index 0000000..a66041a --- /dev/null +++ b/wechat/tip.py @@ -0,0 +1,64 @@ +# -*- coding: utf-8 -*- +# @author : Jared +# @date : 2019/6/22 17:39 + +import json +import time +from datetime import datetime + +import itchat +import requests + + +class CheLaiLe: + count, step, max_count = 0, 1, 90 + + +class ShellIsland: + start = False + + +file_helper = 'filehelper' + + +@itchat.msg_register(itchat.content.TEXT) +def text(msg): + receive_text = msg['Text'] + to_user_id = msg['ToUserName'] # 接收人 + # from_user_id = msg['FromUserName'] # 发送人 + + if to_user_id == file_helper: + if receive_text.startswith("b"): + # 车来了 + CheLaiLe.count = 0 + while CheLaiLe.count <= CheLaiLe.max_count: + CheLaiLe.count += CheLaiLe.step + if datetime.now().hour < 12: + resp = requests.get( + 'https://api.chelaile.net.cn/bus/line!busesDetail.action' + '?sign=3BszHrCL%2BYb8Ix6to1kyJw%3D%3D&v=3.79.2&s=android' + '&cityId=040&targetOrder=6&lineId=020-07730-0').text + else: + resp = requests.get( + 'https://api.chelaile.net.cn/bus/line!busesDetail.action' + '?sign=CMkEe%2BVN716ghdReiM5G9Q%3D%3D&s=android&v=3.79.2' + '&cityId=040&targetOrder=4&lineId=020-07730-1').text + resp = resp.replace("**YGKJ", "").replace("YGKJ##", "") + json_content = json.loads(resp) + data = json_content.get('jsonr').get("data") + tip_desc, pre_arrival_desc = data.get("tip").get("desc"), data.get("depDesc") + + send_msg = "{}\n[{}]".format(tip_desc, pre_arrival_desc) + itchat.send(send_msg, toUserName=file_helper) + time.sleep(30) + + if receive_text.startswith("s"): + # 贝壳小岛 + ShellIsland.start = not ShellIsland.start + while ShellIsland.start: + itchat.send("", toUserName=file_helper) + time.sleep(60) + + +itchat.auto_login(hotReload=True, enableCmdQR=2) +itchat.run() diff --git a/wechat/tipjob.py b/wechat/tipjob.py new file mode 100644 index 0000000..15b9cbe --- /dev/null +++ b/wechat/tipjob.py @@ -0,0 +1,21 @@ +import os +from apscheduler.schedulers.background import * + + +def job_task(): + scheduler = BlockingScheduler() + scheduler.add_job(tip, 'cron', hour=19, minute="30,45,59") + scheduler.start() + + +def tip(): + resp = os.popen("ps -ef|grep python | grep tip.py | wc -l").read() + if int(resp) <= 1: + print("脚本即将运行...") + os.system("nohup python3 tip.py >> /dev/null 2>&1 &") + else: + print("脚本已经运行...") + + +if __name__ == '__main__': + job_task() diff --git a/wechat/wx_itchat.py b/wechat/wx_itchat.py index 59f83d9..44c74fe 100644 --- a/wechat/wx_itchat.py +++ b/wechat/wx_itchat.py @@ -138,7 +138,7 @@ def _handle_province(self, member_count): for province in list(self.num_of_province.keys()): number = self.num_of_province[province] - if number / member_count < .02 and province != self.unknown_province: + if number / member_count < .01 and province != self.unknown_province: other_province_num = self.num_of_province.get(self.unknown_province) self.num_of_province.__setitem__(self.unknown_province, other_province_num + number) del self.num_of_province[province] @@ -150,6 +150,9 @@ def _plt_gender_bar(self, title): # font size include: ‘xx-small’,x-small’,'small’,'medium’,‘large’,‘x-large’,‘xx-large’ or number # plt.xticks(size='small',rotation=30) + + plt.rcParams["font.sans-serif"] = ["SimHei"] + plt.rcParams["axes.unicode_minus"] = False plt.bar('男', self.male_num, color='yellow') plt.bar('女', self.female_num, color='pink') plt.bar('未知', self.unknown_gender, color='gray') @@ -166,6 +169,8 @@ def _plt_province_pie(self, title): data = np.array(list(self.num_of_province.values())) labels = list(self.num_of_province.keys()) + plt.rcParams["font.sans-serif"] = ["SimHei"] + plt.rcParams["axes.unicode_minus"] = False plt.pie(data, labels=labels, autopct='%.1f%%', ) plt.axis('equal') plt.legend(loc=2, prop={'size': 5.5}) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index f5eb24b..fdd4360 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -1,32 +1,33 @@ -import json -import random import re import time +import json +import random +import urllib3 +import requests from bs4 import BeautifulSoup from datetime import datetime -import requests - from utils import pgs, es +urllib3.disable_warnings() wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# def load(): -# for i in range(21, 22): -# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ -# "tb_mps where id = {} and show = True".format(i) -# result = postgres.fetch_all(sql) -# for r in result: -# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ -# r[3], r[4], r[5] -# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( -# r_pass_ticket, r_wap_sid2) -# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) -# mps.start() +def load(): + for i in [13, 14, 25, 3]: + sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ + "tb_mps where id = {} and show = True".format(i) + result = postgres.fetch_all(sql) + for r in result: + r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ + r[3], r[4], r[5] + r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( + r_pass_ticket, r_wap_sid2) + mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) + mps.start() class WxMps(object): @@ -214,7 +215,7 @@ def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id): like_num = comment.get('like_num') # 点赞数 reply_list = comment.get('reply')['reply_list'] # 回复数据 - reply_content, reply_like_num, reply_create_time = None, None, None + reply_content = reply_like_num = reply_create_time = None if reply_list: first_reply = reply_list[0] reply_content = first_reply.get('content') @@ -247,14 +248,14 @@ def _save_article_comment(): if __name__ == '__main__': - _id = 18 - biz = 'MjM5NjAxOTU4MA==' - app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # 开始爬取文章及评论 - wxMps.start() - # load() + # _id = 18 + # biz = 'MjM5NjAxOTU4MA==' + # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # # 开始爬取文章及评论 + # wxMps.start() + load() diff --git a/wechat/wx_mps.sh b/wechat/wx_mps.sh index 317ffb9..b802e48 100644 --- a/wechat/wx_mps.sh +++ b/wechat/wx_mps.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -nohup python3 wx_mps.py >> /dev/null 2>&1 & \ No newline at end of file +nohup python3 wx_mps.py >> mps.log 2>&1 & \ No newline at end of file