From 418703e380c1bdcf5f95b6562be6f13abf08725a Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 21 Mar 2019 10:36:19 +0800 Subject: [PATCH 001/148] update wx_mps --- wechat/wx_mps.py | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 27 insertions(+), 28 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index a842029..5ddcd36 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -9,24 +9,23 @@ from utils import pgs, es +wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) +postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) +elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) -# postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) -# elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# -# -# def load(): -# for i in range(19, 20): -# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ -# "tb_mps where id = {} and show = True".format(i) -# result = postgres.fetch_all(sql) -# for r in result: -# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ -# r[3], r[4], r[5] -# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( -# r_pass_ticket, r_wap_sid2) -# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) -# mps.start() + +def load(): + for i in range(20, 21): + sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ + "tb_mps where id = {} and show = True".format(i) + result = postgres.fetch_all(sql) + for r in result: + r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ + r[3], r[4], r[5] + r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( + r_pass_ticket, r_wap_sid2) + mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) + mps.start() class WxMps(object): @@ -247,14 +246,14 @@ def _save_article_comment(): if __name__ == '__main__': - _id = 18 - biz = 'MjM5NjAxOTU4MA==' - app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # 开始爬取文章及评论 - wxMps.start() - # load() + # _id = 18 + # biz = 'MjM5NjAxOTU4MA==' + # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # # 开始爬取文章及评论 + # wxMps.start() + load() From 1f63fa11f212e945e924599e7bca053de506be34 Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 21 Mar 2019 10:56:41 +0800 Subject: [PATCH 002/148] update wx_mps --- wechat/wx_mps.py | 55 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 5ddcd36..39a048c 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -9,23 +9,24 @@ from utils import pgs, es -wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) -postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) -elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') - -def load(): - for i in range(20, 21): - sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ - "tb_mps where id = {} and show = True".format(i) - result = postgres.fetch_all(sql) - for r in result: - r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ - r[3], r[4], r[5] - r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( - r_pass_ticket, r_wap_sid2) - mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) - mps.start() +# wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) +# postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) +# elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') +# +# +# def load(): +# for i in range(20, 21): +# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ +# "tb_mps where id = {} and show = True".format(i) +# result = postgres.fetch_all(sql) +# for r in result: +# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ +# r[3], r[4], r[5] +# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( +# r_pass_ticket, r_wap_sid2) +# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) +# mps.start() class WxMps(object): @@ -246,14 +247,14 @@ def _save_article_comment(): if __name__ == '__main__': - # _id = 18 - # biz = 'MjM5NjAxOTU4MA==' - # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # # 开始爬取文章及评论 - # wxMps.start() - load() + _id = 18 + biz = 'MjM5NjAxOTU4MA==' + app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # 开始爬取文章及评论 + wxMps.start() + # load() From 1faa55317c150fc518259fad0dc2a5fa270a26da Mon Sep 17 00:00:00 2001 From: happyJared Date: Fri, 22 Mar 2019 13:55:52 +0800 Subject: [PATCH 003/148] =?UTF-8?q?refactor:=20=E8=B0=83=E6=95=B4=E9=A1=B9?= =?UTF-8?q?=E7=9B=AE=E7=BB=93=E6=9E=84,=E8=A1=A5=E5=85=85=E8=AF=B4?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 22 ++++++++++---------- auto/qmm.py | 2 +- myflask/flask-01 | 12 ----------- scrapys/boss/boss/middlewares.py | 8 +++----- scrapys/lagou/lagou/settings.py | 4 ++-- scrapys/lagou/lagou/spiders/job.py | 1 + scrapys/lagou/lagou/spiders/main.py | 5 +---- tsflow/tsflow01.py | 8 -------- utils/__init__.py | 0 utils/banker.py | 22 -------------------- utils/vercode.py | 31 ----------------------------- wechat/__init__.py | 0 wechat/test.py | 0 13 files changed, 18 insertions(+), 97 deletions(-) delete mode 100644 myflask/flask-01 delete mode 100644 tsflow/tsflow01.py delete mode 100644 utils/__init__.py delete mode 100644 utils/banker.py delete mode 100644 utils/vercode.py delete mode 100644 wechat/__init__.py delete mode 100644 wechat/test.py diff --git a/README.md b/README.md index 260341a..8e4a3df 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,13 @@ ## Introduction -- `auto`: 自动化&&抓包，如selenium、appium、mitmproxy -- `basic`: python基础练习 +- `auto`: 自动化 && 抓包 Demo, selenium、appium、mitmproxy +- `basic`: python基础 Demo - `blogs`: 基于web.py的简书数据统计 -- `datascience`: 数据科学，如numpy、pandas、matplotlib -- `mydjango`: Django练习 -- `myflask`: Flask练习 -- `orm`: sqlalchemy和peewee示例 -- `scrapys`: scrapy项目 -- `single`: 某小程序爬虫&&机器人 -- `spider`: 爬虫项目 -- `tsflow`: TensorFlow练习 -- `utils`: 工具包 -- `wechat`: 微信机器人&&公众号爬取 \ No newline at end of file +- `datascience`: 数据科学,numpy、pandas、matplotlib +- `mydjango`: Django Demo +- `orm`: sqlalchemy && peewee Demo +- `scrapys`: scrapy examples +- `single`: 某小程序爬虫 && 机器人 +- `spider`: 简单爬虫examples +- `utils`: 小小工具包 +- `wechat`: 微信机器人 && 公众号抓取 \ No newline at end of file diff --git a/auto/qmm.py b/auto/qmm.py index 778b6f0..5dd14b4 100644 --- a/auto/qmm.py +++ b/auto/qmm.py @@ -172,5 +172,5 @@ def _finance_sign(self, driver): if __name__ == '__main__': - qmm = QMM(sleep=3, months='2', days='1-31') + qmm = QMM(sleep=3, months='2-3', days='1-31') qmm.start() diff --git a/myflask/flask-01 b/myflask/flask-01 deleted file mode 100644 index 27c25f1..0000000 --- a/myflask/flask-01 +++ /dev/null @@ -1,12 +0,0 @@ -from flask import Flask - -app = Flask(__name__) - - -@app.route('/') -def hello_world(): - return '2018 , Hello World !' - - -if __name__ == '__main__': - app.run(debug=True) diff --git a/scrapys/boss/boss/middlewares.py b/scrapys/boss/boss/middlewares.py index 505ccf9..5b89b2a 100644 --- a/scrapys/boss/boss/middlewares.py +++ b/scrapys/boss/boss/middlewares.py @@ -5,15 +5,13 @@ # See documentation in: # https://doc.scrapy.org/en/latest/topics/spider-middleware.html -import time import logging import random -import requests -from utils import mycaptcha +import time + from scrapy import signals -from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware from scrapy.downloadermiddlewares.redirect import RedirectMiddleware -from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware +from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware # from scrapys.nearjob import app diff --git a/scrapys/lagou/lagou/settings.py b/scrapys/lagou/lagou/settings.py index 4b4a856..e9173f9 100644 --- a/scrapys/lagou/lagou/settings.py +++ b/scrapys/lagou/lagou/settings.py @@ -22,12 +22,12 @@ ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) -CONCURRENT_REQUESTS = 2 +CONCURRENT_REQUESTS = 1 # Configure a delay for requests for the same website (default: 0) # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs -DOWNLOAD_DELAY = 2 +DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: # CONCURRENT_REQUESTS_PER_DOMAIN = 16 # CONCURRENT_REQUESTS_PER_IP = 12 diff --git a/scrapys/lagou/lagou/spiders/job.py b/scrapys/lagou/lagou/spiders/job.py index 18cf12b..8f71613 100644 --- a/scrapys/lagou/lagou/spiders/job.py +++ b/scrapys/lagou/lagou/spiders/job.py @@ -46,6 +46,7 @@ def start_requests(self): def parse(self, response): self.logger.warning(response) resp = json.loads(response.body_as_unicode()) + self.logger.info(resp) success = resp.get('success') self.logger.info('1. resp code %s success %s' % (resp.get('code'), success)) if success: diff --git a/scrapys/lagou/lagou/spiders/main.py b/scrapys/lagou/lagou/spiders/main.py index 8e9989f..ece44f0 100644 --- a/scrapys/lagou/lagou/spiders/main.py +++ b/scrapys/lagou/lagou/spiders/main.py @@ -15,7 +15,4 @@ def run(): if __name__ == '__main__': - if len(sys.argv) == 1: - cron.cron_blocking(job=run, day_of_week='4', hour='21') - else: - run() + run() diff --git a/tsflow/tsflow01.py b/tsflow/tsflow01.py deleted file mode 100644 index b095ce1..0000000 --- a/tsflow/tsflow01.py +++ /dev/null @@ -1,8 +0,0 @@ -import tensorflow as tf - -hello = tf.constant('hello', dtype=tf.string) - -with tf.Session() as session: - res = session.run(hello).decode('UTF-8') - print(res) - print(type(res)) diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/utils/banker.py b/utils/banker.py deleted file mode 100644 index 7e4d6da..0000000 --- a/utils/banker.py +++ /dev/null @@ -1,22 +0,0 @@ -import time -import requests - - -def get_gold_price(): - """获取实时黄金价格""" - - url = 'https://www.g-banker.com/price/query' - json = {"queryFlag": 3} - resp = requests.post(url, json=json).json() - if resp.get('success') == 'true': - data = resp.get('data') - realtime_price = data.get('realtime_price') - price = realtime_price / 100 - print("{} {} {}".format(realtime_price, price, type(price))) - return price - - -if __name__ == '__main__': - while True: - get_gold_price() - time.sleep(10) diff --git a/utils/vercode.py b/utils/vercode.py deleted file mode 100644 index 97265fb..0000000 --- a/utils/vercode.py +++ /dev/null @@ -1,31 +0,0 @@ -from PIL import Image -import pytesseract - -pytesseract.pytesseract.tesseract_cmd = "E:\\jared\\Python\\Tesseract-OCR\\tesseract.exe" - - -def recognize_code_image(image_file): - # 打开图片 - image = Image.open(image_file) - # image.show() - # 转为灰度图片 - image_grey = image.convert('L') - # image_grey.show() - # 二值化处理 - table = [] - for i in range(256): - if i < 140: - table.append(0) - else: - table.append(1) - image_bi = image_grey.point(table, '1') - # image_bi.show() - image_bi.save('C:\\Users\\CFL\\Desktop\\captcha1.png') - # 识别验证码 - verify_code = pytesseract.image_to_string(image_bi) - - return verify_code - - -if __name__ == '__main__': - print(recognize_code_image('C:\\Users\\CFL\\Desktop\\captcha.png')) diff --git a/wechat/__init__.py b/wechat/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/wechat/test.py b/wechat/test.py deleted file mode 100644 index e69de29..0000000 From 069aea30b0d533205fe1454a34ef1c82a2778419 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 23 Mar 2019 14:33:40 +0800 Subject: [PATCH 004/148] update wx_mps --- wechat/wx_mps.py | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 39a048c..94a3a59 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -10,23 +10,23 @@ from utils import pgs, es -# wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) -# postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) -# elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# -# -# def load(): -# for i in range(20, 21): -# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ -# "tb_mps where id = {} and show = True".format(i) -# result = postgres.fetch_all(sql) -# for r in result: -# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ -# r[3], r[4], r[5] -# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( -# r_pass_ticket, r_wap_sid2) -# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) -# mps.start() +wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) +postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) +elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') + + +def load(): + for i in range(21, 22): + sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ + "tb_mps where id = {} and show = True".format(i) + result = postgres.fetch_all(sql) + for r in result: + r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ + r[3], r[4], r[5] + r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( + r_pass_ticket, r_wap_sid2) + mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) + mps.start() class WxMps(object): @@ -247,14 +247,14 @@ def _save_article_comment(): if __name__ == '__main__': - _id = 18 - biz = 'MjM5NjAxOTU4MA==' - app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # 开始爬取文章及评论 - wxMps.start() - # load() + # _id = 18 + # biz = 'MjM5NjAxOTU4MA==' + # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # # 开始爬取文章及评论 + # wxMps.start() + load() From 058df2664578fd2ecc3d7879427827d359487fae Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 23 Mar 2019 14:42:34 +0800 Subject: [PATCH 005/148] update wx_mps --- wechat/wx_mps.py | 56 ++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 94a3a59..c593ada 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -10,23 +10,23 @@ from utils import pgs, es -wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) -postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) -elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') - - -def load(): - for i in range(21, 22): - sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ - "tb_mps where id = {} and show = True".format(i) - result = postgres.fetch_all(sql) - for r in result: - r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ - r[3], r[4], r[5] - r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( - r_pass_ticket, r_wap_sid2) - mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) - mps.start() +# wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) +# postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) +# elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') +# +# +# def load(): +# for i in range(21, 22): +# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ +# "tb_mps where id = {} and show = True".format(i) +# result = postgres.fetch_all(sql) +# for r in result: +# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ +# r[3], r[4], r[5] +# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( +# r_pass_ticket, r_wap_sid2) +# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) +# mps.start() class WxMps(object): @@ -247,14 +247,14 @@ def _save_article_comment(): if __name__ == '__main__': - # _id = 18 - # biz = 'MjM5NjAxOTU4MA==' - # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # # 开始爬取文章及评论 - # wxMps.start() - load() + _id = 18 + biz = 'MjM5NjAxOTU4MA==' + app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # 开始爬取文章及评论 + wxMps.start() + # load() From b96f5680534297db62c6f2bcb8aa941363acbd19 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 23 Mar 2019 14:43:10 +0800 Subject: [PATCH 006/148] update wx_mps --- wechat/wx_mps.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index c593ada..f5eb24b 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -10,11 +10,11 @@ from utils import pgs, es -# wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) -# postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) -# elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# -# +wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) +postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) +elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') + + # def load(): # for i in range(21, 22): # sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ From cc69b620a963791a36eb29e2be2c667fc16f16c4 Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 25 Mar 2019 17:52:15 +0800 Subject: [PATCH 007/148] feature: init jianshu spider --- orm/sqlalchemy_crud.py | 2 +- spider/jianshu/spider.py | 90 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 spider/jianshu/spider.py diff --git a/orm/sqlalchemy_crud.py b/orm/sqlalchemy_crud.py index 8238d6b..50bd9e1 100644 --- a/orm/sqlalchemy_crud.py +++ b/orm/sqlalchemy_crud.py @@ -25,7 +25,7 @@ def __str__(self) -> str: # 创建数据库连接（'数据库类型+数据库驱动名称://用户名:密码@ip地址:端口/数据库名'） -conn = "postgresql+psycopg2://postgres:@:5432/postgres" +conn = "postgresql+psycopg2://postgres:@:5432/postgres" engine = create_engine(conn, encoding='UTF-8', echo=False) # echo=True表示输出执行日志,默认为False # 删除映射数据表(如果存在) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py new file mode 100644 index 0000000..10016ee --- /dev/null +++ b/spider/jianshu/spider.py @@ -0,0 +1,90 @@ +import time +import datetime + +import requests +from bs4 import BeautifulSoup +from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +# 创建对象的基类: +Base = declarative_base() + + +class Article(Base): + __tablename__ = 'tb_jianshu_article' + + # 表字段 + id = Column(Integer, primary_key=True, autoincrement=True) + article_url_suffix = Column(String(50), nullable=False, index=True) + ic_paid = Column(FLOAT(2), nullable=False) + ic_like = Column(Integer, nullable=False) + comment_num = Column(Integer, nullable=False) + publish_time = Column(TIMESTAMP, nullable=False) + create_time = Column(TIMESTAMP, nullable=False, default=datetime.datetime.now()) + + +conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" +engine = create_engine(conn, encoding='UTF-8', echo=False) +Base.metadata.create_all(engine) + +Session = sessionmaker(bind=engine) +session = Session() + + +def save_article(*args): + suffix, paid, like, num, p_time = args + article = session.query(Article).filter(Article.article_url_suffix != suffix).first() + if not article: + article = Article(article_url_suffix=suffix, ic_paid=paid, ic_like=like, comment_num=num, publish_time=p_time) + session.add(article) + session.commit() + + +domain = "https://www.jianshu.com" +headers = { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" +} +# 我的关注列表 +following = "https://www.jianshu.com/users/000a530f461c/following" + +while True: + resp = requests.get(following, headers=headers) + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, 'html.parser') + user_list = bs.find("ul", class_="user-list") + avatar_list = user_list.find_all("a", class_="avatar") + + for avatar in avatar_list: + # 具体关注的人 + following = domain + avatar['href'] + resp = requests.get(following, headers=headers) + + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, 'html.parser') + content_list = bs.find_all("div", class_="content") + for content in content_list: + meta = content.find("div", class_="meta") + + # 简书钻数 && 喜欢人次 && 发布时间 + ic_paid = ic_like = 0 + publish_time = None + span_list = meta.find_all("span") + if len(span_list) == 2: + ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] + else: + ic_paid = span_list[0].text.strip() + ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] + publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') + + publish_date = datetime.datetime.date(publish_time) + today_date = datetime.datetime.date(datetime.datetime.now()) + if publish_date == today_date: + # 查看次数 && 评论人次 + a_list = meta.find_all("a") + article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() + + save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + time.sleep(300) From 7ddc8f2be306e8f3e048cd223d79aa99bc710417 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 25 Mar 2019 23:03:05 +0800 Subject: [PATCH 008/148] update jianshu spider --- spider/jianshu/spider.py | 95 ++++++++++++++++++++-------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 10016ee..7f60464 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -7,16 +7,15 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -# 创建对象的基类: -Base = declarative_base() +# 基类 +base = declarative_base() -class Article(Base): +class Article(base): __tablename__ = 'tb_jianshu_article' - # 表字段 id = Column(Integer, primary_key=True, autoincrement=True) - article_url_suffix = Column(String(50), nullable=False, index=True) + article_url_suffix = Column(String(50), nullable=False, index=True, unique=True) ic_paid = Column(FLOAT(2), nullable=False) ic_like = Column(Integer, nullable=False) comment_num = Column(Integer, nullable=False) @@ -24,9 +23,9 @@ class Article(Base): create_time = Column(TIMESTAMP, nullable=False, default=datetime.datetime.now()) -conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" +conn = "postgresql+psycopg2://postgres:pgsmaster5432@193.112.0.219:12432/scrapy" engine = create_engine(conn, encoding='UTF-8', echo=False) -Base.metadata.create_all(engine) +base.metadata.create_all(engine) Session = sessionmaker(bind=engine) session = Session() @@ -34,7 +33,7 @@ class Article(Base): def save_article(*args): suffix, paid, like, num, p_time = args - article = session.query(Article).filter(Article.article_url_suffix != suffix).first() + article = session.query(Article).filter(Article.article_url_suffix == suffix).first() if not article: article = Article(article_url_suffix=suffix, ic_paid=paid, ic_like=like, comment_num=num, publish_time=p_time) session.add(article) @@ -46,45 +45,47 @@ def save_article(*args): "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" } # 我的关注列表 -following = "https://www.jianshu.com/users/000a530f461c/following" +following = "https://www.jianshu.com/users/000a530f461c/following?page={}" while True: - resp = requests.get(following, headers=headers) - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, 'html.parser') - user_list = bs.find("ul", class_="user-list") - avatar_list = user_list.find_all("a", class_="avatar") - - for avatar in avatar_list: - # 具体关注的人 - following = domain + avatar['href'] - resp = requests.get(following, headers=headers) - - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, 'html.parser') - content_list = bs.find_all("div", class_="content") - for content in content_list: - meta = content.find("div", class_="meta") - - # 简书钻数 && 喜欢人次 && 发布时间 - ic_paid = ic_like = 0 - publish_time = None - span_list = meta.find_all("span") - if len(span_list) == 2: - ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] - else: - ic_paid = span_list[0].text.strip() - ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] - publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') - - publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(datetime.datetime.now()) - if publish_date == today_date: - # 查看次数 && 评论人次 - a_list = meta.find_all("a") - article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() - - save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + for page in range(1, 3): + resp = requests.get(following.format(page), headers=headers) + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, 'html.parser') + user_list = bs.find("ul", class_="user-list") + avatar_list = user_list.find_all("a", class_="avatar") + + for avatar in avatar_list: + # 具体关注的人 + following = domain + avatar['href'] + resp = requests.get(following, headers=headers) + + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, 'html.parser') + content_list = bs.find_all("div", class_="content") + for content in content_list[0:1]: + meta = content.find("div", class_="meta") + + # 简书钻数 && 喜欢人次 && 发布时间 + ic_paid = ic_like = 0 + publish_time = None + span_list = meta.find_all("span") + if len(span_list) == 2: + ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] + else: + ic_paid = span_list[0].text.strip() + ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] + publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') + + publish_date = datetime.datetime.date(publish_time) + today_date = datetime.datetime.date(datetime.datetime.now()) + if publish_date == today_date: + # 查看次数 && 评论人次 + a_list = meta.find_all("a") + article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() + + save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + print("sleep...") time.sleep(300) From 8350c0909856aa83dd67b5fdd015214b3d98d557 Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 09:06:47 +0800 Subject: [PATCH 009/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 7f60464..b3986d6 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -23,7 +23,7 @@ class Article(base): create_time = Column(TIMESTAMP, nullable=False, default=datetime.datetime.now()) -conn = "postgresql+psycopg2://postgres:pgsmaster5432@193.112.0.219:12432/scrapy" +conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" engine = create_engine(conn, encoding='UTF-8', echo=False) base.metadata.create_all(engine) @@ -65,27 +65,29 @@ def save_article(*args): html = resp.text bs = BeautifulSoup(html, 'html.parser') content_list = bs.find_all("div", class_="content") - for content in content_list[0:1]: - meta = content.find("div", class_="meta") - - # 简书钻数 && 喜欢人次 && 发布时间 - ic_paid = ic_like = 0 - publish_time = None - span_list = meta.find_all("span") - if len(span_list) == 2: - ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] - else: - ic_paid = span_list[0].text.strip() - ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] - publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') - - publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(datetime.datetime.now()) - if publish_date == today_date: - # 查看次数 && 评论人次 - a_list = meta.find_all("a") - article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() - - save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + + content = content_list[0] + meta = content.find("div", class_="meta") + + # 简书钻数 && 喜欢人次 && 发布时间 + ic_paid = ic_like = 0 + publish_time = None + span_list = meta.find_all("span") + if len(span_list) == 2: + ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] + else: + ic_paid = span_list[0].text.strip() + ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] + publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') + + publish_date = datetime.datetime.date(publish_time) + today_date = datetime.datetime.date(datetime.datetime.now()) + if publish_date == today_date: + # 查看次数 && 评论人次 + a_list = meta.find_all("a") + article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() + + save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + print("sleep...") time.sleep(300) From cc05288d2ea6315ab346974e3f2d79365b55548a Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 10:40:35 +0800 Subject: [PATCH 010/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 92 +++++++++++++++++++++++----------------- 1 file changed, 53 insertions(+), 39 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index b3986d6..96ac5f1 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -1,3 +1,4 @@ +import os import time import datetime @@ -12,19 +13,22 @@ class Article(base): - __tablename__ = 'tb_jianshu_article' + __tablename__ = "tb_jianshu_article" id = Column(Integer, primary_key=True, autoincrement=True) - article_url_suffix = Column(String(50), nullable=False, index=True, unique=True) - ic_paid = Column(FLOAT(2), nullable=False) - ic_like = Column(Integer, nullable=False) - comment_num = Column(Integer, nullable=False) - publish_time = Column(TIMESTAMP, nullable=False) - create_time = Column(TIMESTAMP, nullable=False, default=datetime.datetime.now()) + article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") + ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") + ic_like = Column(Integer, nullable=False, comment="喜欢人次") + ic_reward = Column(Integer, nullable=False, comment="赞赏人次") + comment_num = Column(Integer, nullable=False, comment="评论人次") + publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") + init_time = datetime.datetime.now() + update_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="更新时间") + create_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="创建时间") conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" -engine = create_engine(conn, encoding='UTF-8', echo=False) +engine = create_engine(conn, encoding="UTF-8", echo=False) base.metadata.create_all(engine) Session = sessionmaker(bind=engine) @@ -32,12 +36,18 @@ class Article(base): def save_article(*args): - suffix, paid, like, num, p_time = args + suffix, paid, like, num, p_time, reward = args article = session.query(Article).filter(Article.article_url_suffix == suffix).first() if not article: - article = Article(article_url_suffix=suffix, ic_paid=paid, ic_like=like, comment_num=num, publish_time=p_time) - session.add(article) - session.commit() + # add + article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, + ic_like=like, comment_num=num, publish_time=p_time) + else: + # update + article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num + article.update_time = datetime.datetime.now() + session.add(article) + session.commit() domain = "https://www.jianshu.com" @@ -48,46 +58,50 @@ def save_article(*args): following = "https://www.jianshu.com/users/000a530f461c/following?page={}" while True: - for page in range(1, 3): + for page in range(1, 5): resp = requests.get(following.format(page), headers=headers) if resp.status_code == 200: html = resp.text - bs = BeautifulSoup(html, 'html.parser') + bs = BeautifulSoup(html, "html.parser") user_list = bs.find("ul", class_="user-list") avatar_list = user_list.find_all("a", class_="avatar") for avatar in avatar_list: # 具体关注的人 - following = domain + avatar['href'] - resp = requests.get(following, headers=headers) + following_detail = domain + avatar["href"] + resp = requests.get(following_detail, headers=headers) if resp.status_code == 200: html = resp.text - bs = BeautifulSoup(html, 'html.parser') + bs = BeautifulSoup(html, "html.parser") content_list = bs.find_all("div", class_="content") - content = content_list[0] - meta = content.find("div", class_="meta") - - # 简书钻数 && 喜欢人次 && 发布时间 - ic_paid = ic_like = 0 - publish_time = None - span_list = meta.find_all("span") - if len(span_list) == 2: - ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] - else: - ic_paid = span_list[0].text.strip() - ic_like, publish_time = span_list[1].text.strip(), span_list[2]["data-shared-at"] - publish_time = datetime.datetime.strptime(publish_time, '%Y-%m-%dT%H:%M:%S+08:00') - - publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(datetime.datetime.now()) - if publish_date == today_date: - # 查看次数 && 评论人次 - a_list = meta.find_all("a") - article_url_suffix, comment_num = a_list[0]['href'], a_list[1].text.strip() - - save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time) + for content in content_list: + meta = content.find("div", class_="meta") + + # 简书钻数 && 喜欢人次 && 发布时间 && 赞赏人次 + ic_paid = ic_like = ic_reward = 0 + publish_time = None + span_list = meta.find_all("span") + span_list_length = len(span_list) + if span_list_length == 2: + ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] + else: + ic_paid, ic_like = span_list[0].text.strip(), span_list[1].text.strip() + if span_list_length == 3: + publish_time = span_list[2]["data-shared-at"] + elif span_list_length == 4: + ic_reward, publish_time = span_list[2].text.strip(), span_list[3]["data-shared-at"] + publish_time = datetime.datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") + + publish_date = datetime.datetime.date(publish_time) + today_date = datetime.datetime.date(datetime.datetime.now()) + if publish_date == today_date: + # 查看次数 && 评论人次 + a_list = meta.find_all("a") + article_url_suffix, comment_num = a_list[0]["href"], a_list[1].text.strip() + + save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time, ic_reward) print("sleep...") time.sleep(300) From aac0854ba9b5b9be523caefeb58ddeb820f5f331 Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 10:54:01 +0800 Subject: [PATCH 011/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 202 ++++++++++++++++++++------------------- 1 file changed, 104 insertions(+), 98 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 96ac5f1..eda0ffb 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -1,6 +1,8 @@ +import sys import os -import time import datetime +import random +import time import requests from bs4 import BeautifulSoup @@ -8,100 +10,104 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -# 基类 -base = declarative_base() - - -class Article(base): - __tablename__ = "tb_jianshu_article" - - id = Column(Integer, primary_key=True, autoincrement=True) - article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") - ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") - ic_like = Column(Integer, nullable=False, comment="喜欢人次") - ic_reward = Column(Integer, nullable=False, comment="赞赏人次") - comment_num = Column(Integer, nullable=False, comment="评论人次") - publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") - init_time = datetime.datetime.now() - update_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="更新时间") - create_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="创建时间") - - -conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" -engine = create_engine(conn, encoding="UTF-8", echo=False) -base.metadata.create_all(engine) - -Session = sessionmaker(bind=engine) -session = Session() - - -def save_article(*args): - suffix, paid, like, num, p_time, reward = args - article = session.query(Article).filter(Article.article_url_suffix == suffix).first() - if not article: - # add - article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, - ic_like=like, comment_num=num, publish_time=p_time) - else: - # update - article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num - article.update_time = datetime.datetime.now() - session.add(article) - session.commit() - - -domain = "https://www.jianshu.com" -headers = { - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" -} -# 我的关注列表 -following = "https://www.jianshu.com/users/000a530f461c/following?page={}" - -while True: - for page in range(1, 5): - resp = requests.get(following.format(page), headers=headers) - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, "html.parser") - user_list = bs.find("ul", class_="user-list") - avatar_list = user_list.find_all("a", class_="avatar") - - for avatar in avatar_list: - # 具体关注的人 - following_detail = domain + avatar["href"] - resp = requests.get(following_detail, headers=headers) - - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, "html.parser") - content_list = bs.find_all("div", class_="content") - - for content in content_list: - meta = content.find("div", class_="meta") - - # 简书钻数 && 喜欢人次 && 发布时间 && 赞赏人次 - ic_paid = ic_like = ic_reward = 0 - publish_time = None - span_list = meta.find_all("span") - span_list_length = len(span_list) - if span_list_length == 2: - ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] - else: - ic_paid, ic_like = span_list[0].text.strip(), span_list[1].text.strip() - if span_list_length == 3: - publish_time = span_list[2]["data-shared-at"] - elif span_list_length == 4: - ic_reward, publish_time = span_list[2].text.strip(), span_list[3]["data-shared-at"] - publish_time = datetime.datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") - - publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(datetime.datetime.now()) - if publish_date == today_date: - # 查看次数 && 评论人次 - a_list = meta.find_all("a") - article_url_suffix, comment_num = a_list[0]["href"], a_list[1].text.strip() - - save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time, ic_reward) - - print("sleep...") - time.sleep(300) +argv_length = sys.argv +if len(argv_length) > 1: + os.system("nohup python3 {} >> /dev/null 2>&1 &".format(sys.argv[0])) +else: + # 基类 + base = declarative_base() + + + class Article(base): + __tablename__ = "tb_jianshu_article" + + id = Column(Integer, primary_key=True, autoincrement=True) + article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") + ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") + ic_like = Column(Integer, nullable=False, comment="喜欢人次") + ic_reward = Column(Integer, nullable=False, comment="赞赏人次") + comment_num = Column(Integer, nullable=False, comment="评论人次") + publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") + init_time = datetime.datetime.now() + update_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="更新时间") + create_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="创建时间") + + + conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" + engine = create_engine(conn, encoding="UTF-8", echo=False) + base.metadata.create_all(engine) + + Session = sessionmaker(bind=engine) + session = Session() + + + def save_article(*args): + suffix, paid, like, num, p_time, reward = args + article = session.query(Article).filter(Article.article_url_suffix == suffix).first() + if not article: + # add + article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, + ic_like=like, comment_num=num, publish_time=p_time) + else: + # update + article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num + article.update_time = datetime.datetime.now() + session.add(article) + session.commit() + + + domain = "https://www.jianshu.com" + headers = { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" + } + # 我的关注列表 + following = "https://www.jianshu.com/users/000a530f461c/following?page={}" + + while True: + for page in range(1, 5): + resp = requests.get(following.format(page), headers=headers) + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, "html.parser") + user_list = bs.find("ul", class_="user-list") + avatar_list = user_list.find_all("a", class_="avatar") + + for avatar in avatar_list: + # 具体关注的人 + following_detail = domain + avatar["href"] + resp = requests.get(following_detail, headers=headers) + + if resp.status_code == 200: + html = resp.text + bs = BeautifulSoup(html, "html.parser") + content_list = bs.find_all("div", class_="content") + + for content in content_list: + meta = content.find("div", class_="meta") + + # 简书钻数 && 喜欢人次 && 发布时间 && 赞赏人次 + ic_paid = ic_like = ic_reward = 0 + publish_time = None + span_list = meta.find_all("span") + span_list_length = len(span_list) + if span_list_length == 2: + ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] + else: + ic_paid, ic_like = span_list[0].text.strip(), span_list[1].text.strip() + if span_list_length == 3: + publish_time = span_list[2]["data-shared-at"] + elif span_list_length == 4: + ic_reward, publish_time = span_list[2].text.strip(), span_list[3]["data-shared-at"] + publish_time = datetime.datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") + + publish_date = datetime.datetime.date(publish_time) + today_date = datetime.datetime.date(datetime.datetime.now()) + if publish_date == today_date: + # 查看次数 && 评论人次 + a_list = meta.find_all("a") + article_url_suffix, comment_num = a_list[0]["href"], a_list[1].text.strip() + + save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time, ic_reward) + + print("sleep...") + time.sleep(random.randint(300, 600)) From c3361cf4f959d45aab8b28db3fde39021c62832d Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 11:41:57 +0800 Subject: [PATCH 012/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index eda0ffb..8ff850a 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -10,9 +10,9 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -argv_length = sys.argv -if len(argv_length) > 1: - os.system("nohup python3 {} >> /dev/null 2>&1 &".format(sys.argv[0])) +argv = sys.argv +if len(argv) > 1: + os.system("nohup python3 {} >> /dev/null 2>&1 &".format(argv[0])) else: # 基类 base = declarative_base() From e561024a8e3a93d96cdc10e5b2d2f6212a3b1bad Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 11:43:54 +0800 Subject: [PATCH 013/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 8ff850a..810f187 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -1,7 +1,7 @@ -import sys -import os import datetime +import os import random +import sys import time import requests @@ -18,6 +18,10 @@ base = declarative_base() + def get_now(): + return datetime.datetime.now() + + class Article(base): __tablename__ = "tb_jianshu_article" @@ -28,7 +32,7 @@ class Article(base): ic_reward = Column(Integer, nullable=False, comment="赞赏人次") comment_num = Column(Integer, nullable=False, comment="评论人次") publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") - init_time = datetime.datetime.now() + init_time = get_now() update_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="更新时间") create_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="创建时间") @@ -51,7 +55,7 @@ def save_article(*args): else: # update article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num - article.update_time = datetime.datetime.now() + article.update_time = get_now() session.add(article) session.commit() @@ -101,7 +105,7 @@ def save_article(*args): publish_time = datetime.datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(datetime.datetime.now()) + today_date = datetime.datetime.date(get_now()) if publish_date == today_date: # 查看次数 && 评论人次 a_list = meta.find_all("a") From ac1bd2cf582bfa253adf41f7ec2aeb2551cb1cd5 Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 11:46:02 +0800 Subject: [PATCH 014/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 810f187..c174fbc 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -1,8 +1,8 @@ -import datetime import os import random import sys import time +from datetime import datetime import requests from bs4 import BeautifulSoup @@ -19,7 +19,7 @@ def get_now(): - return datetime.datetime.now() + return datetime.now() class Article(base): @@ -102,10 +102,10 @@ def save_article(*args): publish_time = span_list[2]["data-shared-at"] elif span_list_length == 4: ic_reward, publish_time = span_list[2].text.strip(), span_list[3]["data-shared-at"] - publish_time = datetime.datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") + publish_time = datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") - publish_date = datetime.datetime.date(publish_time) - today_date = datetime.datetime.date(get_now()) + publish_date = datetime.date(publish_time) + today_date = datetime.date(get_now()) if publish_date == today_date: # 查看次数 && 评论人次 a_list = meta.find_all("a") From aea10cfaf9a97e040b042e6fb2bd525c676810a5 Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 13:26:06 +0800 Subject: [PATCH 015/148] refactor: update jianshu spider --- spider/jianshu/spider.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index c174fbc..729616b 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -32,9 +32,8 @@ class Article(base): ic_reward = Column(Integer, nullable=False, comment="赞赏人次") comment_num = Column(Integer, nullable=False, comment="评论人次") publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") - init_time = get_now() - update_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="更新时间") - create_time = Column(TIMESTAMP, nullable=False, default=init_time, comment="创建时间") + update_time = Column(TIMESTAMP, nullable=False, comment="更新时间") + create_time = Column(TIMESTAMP, nullable=False, comment="创建时间") conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" @@ -50,8 +49,9 @@ def save_article(*args): article = session.query(Article).filter(Article.article_url_suffix == suffix).first() if not article: # add - article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, - ic_like=like, comment_num=num, publish_time=p_time) + init_time = get_now() + article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, ic_like=like, + comment_num=num, publish_time=p_time, update_time=init_time, create_time=init_time) else: # update article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num From 39704ed75fd6c3833078e36532dffbfd23bd73b0 Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 15:03:23 +0800 Subject: [PATCH 016/148] feature: update jianshu spider --- spider/jianshu/robot.py | 85 ++++++++++++++++++++++++++++++++++++++++ spider/jianshu/spider.py | 3 +- 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 spider/jianshu/robot.py diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py new file mode 100644 index 0000000..9aa6ce6 --- /dev/null +++ b/spider/jianshu/robot.py @@ -0,0 +1,85 @@ +import os +import sys +import requests +from datetime import datetime + +from bs4 import BeautifulSoup +from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, BOOLEAN, create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker + +argv = sys.argv +if len(argv) > 1: + os.system("nohup python3 {} >> /dev/null 2>&1 &".format(argv[0])) +else: + # 基类 + base = declarative_base() + + + def get_now(): + return datetime.now() + + + class Article(base): + __tablename__ = "tb_jianshu_article" + + id = Column(Integer, primary_key=True, autoincrement=True) + article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") + ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") + ic_like = Column(Integer, nullable=False, comment="喜欢人次") + ic_reward = Column(Integer, nullable=False, comment="赞赏人次") + comment_num = Column(Integer, nullable=False, comment="评论人次") + publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") + processed = Column(BOOLEAN, nullable=False, default=False, comment="处理状态。True:已处理;False:未处理") + update_time = Column(TIMESTAMP, nullable=False, comment="更新时间") + create_time = Column(TIMESTAMP, nullable=False, comment="创建时间") + + + class Info(base): + __tablename__ = "tb_jianshu_info" + + id = Column(Integer, primary_key=True, autoincrement=True) + cookie = Column(String(3000)) + + + conn = "postgresql+psycopg2://postgres:pgsmaster5432@193.112.0.219:12432/scrapy" + engine = create_engine(conn, encoding="UTF-8", echo=False) + base.metadata.create_all(engine) + + Session = sessionmaker(bind=engine) + session = Session() + + domain = "https://www.jianshu.com" + headers = { + "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" + } + + + def get_info(): + return session.query(Info).first() + + + def get_article(): + return session.query(Article).filter_by(processed=False).all() + + + def update_article(article_id): + article = session.query(Article).get(article_id) + article.processed = True + session.add(article) + session.commit() + + + def may_like(): + for article in get_article(): + info = get_info() + headers["cookie"] = info.cookie + article_detail = domain + article.article_url_suffix + resp = requests.get(article_detail, headers=headers) + if resp.status_code == 200: + bs = BeautifulSoup(resp.text, "html.parser") + meta = bs.find("meta", {"name": "csrf-token"}) + token = meta["content"] + + + may_like() diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py index 729616b..1009bd0 100644 --- a/spider/jianshu/spider.py +++ b/spider/jianshu/spider.py @@ -6,7 +6,7 @@ import requests from bs4 import BeautifulSoup -from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, create_engine +from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, BOOLEAN, create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker @@ -32,6 +32,7 @@ class Article(base): ic_reward = Column(Integer, nullable=False, comment="赞赏人次") comment_num = Column(Integer, nullable=False, comment="评论人次") publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") + processed = Column(BOOLEAN, nullable=False, default=False, comment="处理状态。True:已处理;False:未处理") update_time = Column(TIMESTAMP, nullable=False, comment="更新时间") create_time = Column(TIMESTAMP, nullable=False, comment="创建时间") From d3bc98f761d06040dc1c4154bb73cbe9a5a8769d Mon Sep 17 00:00:00 2001 From: happyJared Date: Tue, 26 Mar 2019 15:18:33 +0800 Subject: [PATCH 017/148] feature: update jianshu spider --- spider/jianshu/robot.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 9aa6ce6..c687fbe 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -50,6 +50,7 @@ class Info(base): session = Session() domain = "https://www.jianshu.com" + note_like = "https://www.jianshu.com/notes/{}/like" headers = { "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" } @@ -79,7 +80,12 @@ def may_like(): if resp.status_code == 200: bs = BeautifulSoup(resp.text, "html.parser") meta = bs.find("meta", {"name": "csrf-token"}) - token = meta["content"] + note = bs.find("div", {"data-vcomp": "recommended-notes"}) + + note_id = note["data-note-id"] + headers["x-csrf-token"] = meta["content"] + resp = requests.post(note_like.format(note_id), headers) + print(resp.status_code) may_like() From c0c4ea6dceb159eb66781f3767ebe57ed5b542a7 Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 27 Mar 2019 11:44:16 +0800 Subject: [PATCH 018/148] feature: init bihu robot --- spider/bihu/job.py | 15 ++++++++ spider/bihu/robot.py | 85 +++++++++++++++++++++++++++++++++++++++++ spider/bihu/start.sh | 2 + spider/jianshu/robot.py | 18 +++++++-- 4 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 spider/bihu/job.py create mode 100644 spider/bihu/robot.py create mode 100644 spider/bihu/start.sh diff --git a/spider/bihu/job.py b/spider/bihu/job.py new file mode 100644 index 0000000..c83b3e5 --- /dev/null +++ b/spider/bihu/job.py @@ -0,0 +1,15 @@ +import os +import sys + +from apscheduler.schedulers.background import BlockingScheduler + +argv = sys.argv + + +def job(): + os.system("nohup python3 robot.py {} {} >> /dev/null 2>&1 &".format(argv[1], argv[2])) + + +scheduler = BlockingScheduler() +scheduler.add_job(job, 'bihu', hour=7) +scheduler.start() diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py new file mode 100644 index 0000000..0e219cf --- /dev/null +++ b/spider/bihu/robot.py @@ -0,0 +1,85 @@ +import os +import sys +import time +import random +from bs4 import BeautifulSoup +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +argv = sys.argv +argv_length = len(argv) + + +def sleep(): + """随机休眠""" + time.sleep(random.randint(3, 10)) + + +# 无头模式 +options = Options() +options.add_argument('window-size=1366x728') +options.add_argument('--disable-gpu') +options.add_argument('--hide-scrollbars') +options.add_argument('--no-sandbox') +options.add_argument('--disable-dev-shm-usage') +options.add_argument('--headless') +driver = webdriver.Chrome(chrome_options=options) + +# 登录币乎 +bihu = "https://bihu.com" +bihu_login = "{}/login".format(bihu) +driver.get(bihu_login) + +driver.maximize_window() + +username = argv[1] if argv_length > 1 else input("请输入用户名\n") +driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[1]/input').send_keys( + username) +password = argv[2] if argv_length > 2 else input("请输入登录密码\n") +driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[2]/input').send_keys( + password) + +driver.find_element_by_class_name("LoaderButton").click() +sleep() + +print(driver.title) +# 登录成功 +if driver.current_url == bihu: + + bs = BeautifulSoup(driver.page_source, 'html.parser') + item_list = bs.find_all("li", class_="ArticleItem") + articles = {} + for article_item in item_list: + # 解析ArticleItem + user_info = article_item.find("div", class_="user-info") + p_list = user_info.find_all("p") + publish_time = p_list[-1].text + + bottom = article_item.find("div", class_="item-bottom") + bottom_list = bottom.find_all("span") + # Key数 && 点赞数 && 评论数 + key, like, comment = bottom_list + key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + + content_info = article_item.find("div", class_="content-info") + info = content_info.find("a") + articles[info["href"]] = int(like_num) + + num = 0 + # 点赞操作 + for key, value in sorted(articles.items(), key=lambda item: item[1]): + if num >= 6: break + + article = "{}{}".format(bihu, key) + driver.get(article) + sleep() + target = driver.find_element_by_class_name("collect-div") + driver.execute_script("arguments[0].scrollIntoView()", target) + sleep() + + details = driver.find_element_by_class_name("article-details-center2") + buttons = details.find_elements_by_tag_name("button") + if len(buttons) == 3: + num += 1 + buttons[1].click() + sleep() diff --git a/spider/bihu/start.sh b/spider/bihu/start.sh new file mode 100644 index 0000000..2225524 --- /dev/null +++ b/spider/bihu/start.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +nohup python3 job.py xx xx >> /dev/null 2>&1 & \ No newline at end of file diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index c687fbe..5fbdcff 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -10,8 +10,10 @@ argv = sys.argv if len(argv) > 1: + # python3 robot.py jianshu os.system("nohup python3 {} >> /dev/null 2>&1 &".format(argv[0])) else: + # python3 robot.py # 基类 base = declarative_base() @@ -39,10 +41,11 @@ class Info(base): __tablename__ = "tb_jianshu_info" id = Column(Integer, primary_key=True, autoincrement=True) - cookie = Column(String(3000)) + cookie = Column(String(3000), nullable=False) + expired = Column(BOOLEAN, nullable=False) - conn = "postgresql+psycopg2://postgres:pgsmaster5432@193.112.0.219:12432/scrapy" + conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" engine = create_engine(conn, encoding="UTF-8", echo=False) base.metadata.create_all(engine) @@ -60,6 +63,13 @@ def get_info(): return session.query(Info).first() + def update_info(): + info = session.query(Info).first() + info.expired = True + session.add(info) + session.commit() + + def get_article(): return session.query(Article).filter_by(processed=False).all() @@ -85,7 +95,9 @@ def may_like(): note_id = note["data-note-id"] headers["x-csrf-token"] = meta["content"] resp = requests.post(note_like.format(note_id), headers) - print(resp.status_code) + + if not resp.status_code == 200: + update_info() may_like() From 6b97f23f1da4ee9e85c00affd7f9360e2ee86bbc Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 27 Mar 2019 13:23:20 +0800 Subject: [PATCH 019/148] feature: init bihu robot --- spider/bihu/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/bihu/job.py b/spider/bihu/job.py index c83b3e5..30a7f76 100644 --- a/spider/bihu/job.py +++ b/spider/bihu/job.py @@ -11,5 +11,5 @@ def job(): scheduler = BlockingScheduler() -scheduler.add_job(job, 'bihu', hour=7) +scheduler.add_job(job, "cron", hour=7) scheduler.start() From 4223d9453c9b3f4db0b6d791cdcc41dfe3f79032 Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 28 Mar 2019 09:13:45 +0800 Subject: [PATCH 020/148] fix: fix bihu robot --- spider/bihu/robot.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py index 0e219cf..4f4c154 100644 --- a/spider/bihu/robot.py +++ b/spider/bihu/robot.py @@ -19,7 +19,6 @@ def sleep(): options = Options() options.add_argument('window-size=1366x728') options.add_argument('--disable-gpu') -options.add_argument('--hide-scrollbars') options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--headless') @@ -42,9 +41,9 @@ def sleep(): driver.find_element_by_class_name("LoaderButton").click() sleep() -print(driver.title) +print(driver.current_url) # 登录成功 -if driver.current_url == bihu: +if driver.current_url == bihu + "/": bs = BeautifulSoup(driver.page_source, 'html.parser') item_list = bs.find_all("li", class_="ArticleItem") From a61823b2ea91aad0eb71ad77628a0a2e0de58e02 Mon Sep 17 00:00:00 2001 From: happyJared Date: Fri, 29 Mar 2019 16:52:09 +0800 Subject: [PATCH 021/148] refactor: update bihu robot --- spider/bihu/job.py | 15 ------ spider/bihu/robot.py | 122 ++++++++++++++++++++++++++++--------------- spider/bihu/start.sh | 2 - 3 files changed, 80 insertions(+), 59 deletions(-) delete mode 100644 spider/bihu/job.py delete mode 100644 spider/bihu/start.sh diff --git a/spider/bihu/job.py b/spider/bihu/job.py deleted file mode 100644 index 30a7f76..0000000 --- a/spider/bihu/job.py +++ /dev/null @@ -1,15 +0,0 @@ -import os -import sys - -from apscheduler.schedulers.background import BlockingScheduler - -argv = sys.argv - - -def job(): - os.system("nohup python3 robot.py {} {} >> /dev/null 2>&1 &".format(argv[1], argv[2])) - - -scheduler = BlockingScheduler() -scheduler.add_job(job, "cron", hour=7) -scheduler.start() diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py index 4f4c154..53d1045 100644 --- a/spider/bihu/robot.py +++ b/spider/bihu/robot.py @@ -1,7 +1,7 @@ -import os +import random import sys import time -import random + from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options @@ -10,9 +10,9 @@ argv_length = len(argv) -def sleep(): +def sleep(min_seconds=3, max_seconds=10): """随机休眠""" - time.sleep(random.randint(3, 10)) + time.sleep(random.randint(min_seconds, max_seconds)) # 无头模式 @@ -41,44 +41,82 @@ def sleep(): driver.find_element_by_class_name("LoaderButton").click() sleep() -print(driver.current_url) # 登录成功 +print(driver.current_url) if driver.current_url == bihu + "/": - bs = BeautifulSoup(driver.page_source, 'html.parser') - item_list = bs.find_all("li", class_="ArticleItem") - articles = {} - for article_item in item_list: - # 解析ArticleItem - user_info = article_item.find("div", class_="user-info") - p_list = user_info.find_all("p") - publish_time = p_list[-1].text - - bottom = article_item.find("div", class_="item-bottom") - bottom_list = bottom.find_all("span") - # Key数 && 点赞数 && 评论数 - key, like, comment = bottom_list - key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() - - content_info = article_item.find("div", class_="content-info") - info = content_info.find("a") - articles[info["href"]] = int(like_num) - - num = 0 - # 点赞操作 - for key, value in sorted(articles.items(), key=lambda item: item[1]): - if num >= 6: break - - article = "{}{}".format(bihu, key) - driver.get(article) - sleep() - target = driver.find_element_by_class_name("collect-div") - driver.execute_script("arguments[0].scrollIntoView()", target) - sleep() - - details = driver.find_element_by_class_name("article-details-center2") - buttons = details.find_elements_by_tag_name("button") - if len(buttons) == 3: - num += 1 - buttons[1].click() - sleep() + # # 点赞长文 + # bs = BeautifulSoup(driver.page_source, 'html.parser') + # item_list = bs.find_all("li", class_="ArticleItem") + # articles = {} + # for article_item in item_list: + # # 解析ArticleItem + # user_info = article_item.find("div", class_="user-info") + # p_list = user_info.find_all("p") + # publish_time = p_list[-1].text + # + # bottom = article_item.find("div", class_="item-bottom") + # bottom_list = bottom.find_all("span") + # # Key数 && 点赞数 && 评论数 + # key, like, comment = bottom_list + # key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + # + # content_info = article_item.find("div", class_="content-info") + # info = content_info.find("a") + # articles[info["href"]] = int(like_num) + # + # num = 0 + # # 点赞操作 + # for key, value in sorted(articles.items(), key=lambda item: item[1]): + # if num >= 6: break + # + # article = "{}{}".format(bihu, key) + # driver.get(article) + # sleep() + # target = driver.find_element_by_class_name("collect-div") + # driver.execute_script("arguments[0].scrollIntoView()", target) + # sleep() + # + # details = driver.find_element_by_class_name("article-details-center2") + # buttons = details.find_elements_by_tag_name("button") + # if len(buttons) == 3: + # num += 1 + # buttons[1].click() + # sleep() + # + # print("长文已点赞完...") + + # 点赞微文 + driver.get("https://bihu.com/shortcontents") + sleep() + i = 0 + while True: + driver.execute_script("document.body.scrollTop=100000") + sleep(3, 5) + i += 1 + if i >= 5: break + + short_items = driver.find_elements_by_class_name("ShortItem") + for short_item in short_items: + bottom = short_item.find_element_by_class_name("item-bottom") + buttons = bottom.find_elements_by_tag_name("button") + buttons[1].click() + sleep(1, 3) + + # bs = BeautifulSoup(driver.page_source, 'html.parser') + # item_list = bs.find_all("li", class_="ShortItem") + # num = 0 + # for article_item in item_list: + # if num >= 100: break + # # 解析ShortItem + # user_info = article_item.find("div", class_="user-info") + # p_list = user_info.find_all("p") + # publish_time = p_list[-1].text + # + # bottom = article_item.find("div", class_="item-bottom") + # bottom_list = bottom.find_all("span") + # # Key数 && 点赞数 && 评论数 + # key, like, comment = bottom_list + # key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + + print("微文已点赞完...") diff --git a/spider/bihu/start.sh b/spider/bihu/start.sh deleted file mode 100644 index 2225524..0000000 --- a/spider/bihu/start.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -nohup python3 job.py xx xx >> /dev/null 2>&1 & \ No newline at end of file From 4bdf9677f7c878c9b71e823298687a142432145f Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 1 Apr 2019 09:25:58 +0800 Subject: [PATCH 022/148] refactor: update bihu robot --- spider/bihu/robot.py | 109 ++++++++++++++++++++++--------------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py index 53d1045..e641fda 100644 --- a/spider/bihu/robot.py +++ b/spider/bihu/robot.py @@ -45,63 +45,66 @@ def sleep(min_seconds=3, max_seconds=10): print(driver.current_url) if driver.current_url == bihu + "/": - # # 点赞长文 - # bs = BeautifulSoup(driver.page_source, 'html.parser') - # item_list = bs.find_all("li", class_="ArticleItem") - # articles = {} - # for article_item in item_list: - # # 解析ArticleItem - # user_info = article_item.find("div", class_="user-info") - # p_list = user_info.find_all("p") - # publish_time = p_list[-1].text - # - # bottom = article_item.find("div", class_="item-bottom") - # bottom_list = bottom.find_all("span") - # # Key数 && 点赞数 && 评论数 - # key, like, comment = bottom_list - # key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() - # - # content_info = article_item.find("div", class_="content-info") - # info = content_info.find("a") - # articles[info["href"]] = int(like_num) - # - # num = 0 - # # 点赞操作 - # for key, value in sorted(articles.items(), key=lambda item: item[1]): - # if num >= 6: break - # - # article = "{}{}".format(bihu, key) - # driver.get(article) - # sleep() - # target = driver.find_element_by_class_name("collect-div") - # driver.execute_script("arguments[0].scrollIntoView()", target) - # sleep() - # - # details = driver.find_element_by_class_name("article-details-center2") - # buttons = details.find_elements_by_tag_name("button") - # if len(buttons) == 3: - # num += 1 - # buttons[1].click() - # sleep() - # - # print("长文已点赞完...") - - # 点赞微文 + # 1. 点赞长文 + bs = BeautifulSoup(driver.page_source, 'html.parser') + item_list = bs.find_all("li", class_="ArticleItem") + articles = {} + for article_item in item_list: + # 解析ArticleItem + user_info = article_item.find("div", class_="user-info") + p_list = user_info.find_all("p") + publish_time = p_list[-1].text + + bottom = article_item.find("div", class_="item-bottom") + bottom_list = bottom.find_all("span") + # Key数 && 点赞数 && 评论数 + key, like, comment = bottom_list + key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip() + + content_info = article_item.find("div", class_="content-info") + info = content_info.find("a") + articles[info["href"]] = int(like_num) + + num = 0 + # 点赞操作 + for key, value in sorted(articles.items(), key=lambda item: item[1]): + if num >= 6: break + + article = "{}{}".format(bihu, key) + driver.get(article) + sleep() + target = driver.find_element_by_class_name("collect-div") + driver.execute_script("arguments[0].scrollIntoView()", target) + sleep() + + details = driver.find_element_by_class_name("article-details-center2") + buttons = details.find_elements_by_tag_name("button") + if len(buttons) == 3: + num += 1 + buttons[1].click() + sleep() + + print("长文已点赞完...") + + # 2. 点赞微文 driver.get("https://bihu.com/shortcontents") sleep() + i = 0 while True: - driver.execute_script("document.body.scrollTop=100000") - sleep(3, 5) - i += 1 - if i >= 5: break - - short_items = driver.find_elements_by_class_name("ShortItem") - for short_item in short_items: - bottom = short_item.find_element_by_class_name("item-bottom") - buttons = bottom.find_elements_by_tag_name("button") - buttons[1].click() - sleep(1, 3) + short_items = driver.find_elements_by_class_name("ShortItem") + short_items_length = len(short_items) + for index, short_item in enumerate(short_items): + bottom = short_item.find_element_by_class_name("item-bottom") + buttons = bottom.find_elements_by_tag_name("button") + like_button = buttons[1] + if like_button and like_button.is_enabled(): + like_button.click() + sleep(1, 3) + i += 1 + if index + 1 == short_items_length: + sleep(5, 10) + if i >= 100: break # bs = BeautifulSoup(driver.page_source, 'html.parser') # item_list = bs.find_all("li", class_="ShortItem") From 412bd387be019439f8e47f19a0904c131541ce26 Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 1 Apr 2019 09:56:16 +0800 Subject: [PATCH 023/148] refactor: update qmm robot --- auto/qmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto/qmm.py b/auto/qmm.py index 5dd14b4..7591ca2 100644 --- a/auto/qmm.py +++ b/auto/qmm.py @@ -172,5 +172,5 @@ def _finance_sign(self, driver): if __name__ == '__main__': - qmm = QMM(sleep=3, months='2-3', days='1-31') + qmm = QMM(sleep=3, months='4', days='1-30') qmm.start() From 2dccd65005b6e83c907db43c541fd2fc98e6c5b9 Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 1 Apr 2019 17:12:10 +0800 Subject: [PATCH 024/148] refactor: Perfect usage --- wechat/wx_mps.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index f5eb24b..046dc95 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -9,7 +9,6 @@ from utils import pgs, es - wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') @@ -214,7 +213,7 @@ def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id): like_num = comment.get('like_num') # 点赞数 reply_list = comment.get('reply')['reply_list'] # 回复数据 - reply_content, reply_like_num, reply_create_time = None, None, None + reply_content = reply_like_num = reply_create_time = None if reply_list: first_reply = reply_list[0] reply_content = first_reply.get('content') From 23f80cafe8fc7b22d3f69bb736f01926b60d089c Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 1 Apr 2019 17:14:32 +0800 Subject: [PATCH 025/148] refactor: Perfect usage --- wechat/wx_mps.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 046dc95..1c7d36f 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -1,14 +1,16 @@ -import json -import random import re import time +import json +import random +import urllib3 +import requests from bs4 import BeautifulSoup from datetime import datetime -import requests - from utils import pgs, es +urllib3.disable_warnings() + wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的) postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps) elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') From 95daf3d50bdf86a03c3e3167f81ffecc86e2f8aa Mon Sep 17 00:00:00 2001 From: happyJared Date: Mon, 1 Apr 2019 17:45:37 +0800 Subject: [PATCH 026/148] test --- scrapys/music163/music163/util/test.py | 185 +++++++++++++++++++++++++ 1 file changed, 185 insertions(+) diff --git a/scrapys/music163/music163/util/test.py b/scrapys/music163/music163/util/test.py index e69de29..26a2fde 100644 --- a/scrapys/music163/music163/util/test.py +++ b/scrapys/music163/music163/util/test.py @@ -0,0 +1,185 @@ +import json +import random +import re +import time +import urllib3 +from datetime import datetime + +import requests +from bs4 import BeautifulSoup + +urllib3.disable_warnings() + + +class WxMps(object): + """微信公众号文章、评论抓取爬虫""" + + def __init__(self, _mps_id, _biz, _pass_ticket, _app_msg_token, _cookie, last_msg_id=0, _offset=0): + self.offset = _offset + self.mps_id = _mps_id + self.last_msg_id = last_msg_id # 上次抓取的最后消息的id + self.biz = _biz # 公众号标志 + self.msg_token = _app_msg_token # 票据(非固定) + self.pass_ticket = _pass_ticket # 票据(非固定) + self.headers = { + 'Cookie': _cookie, # Cookie(非固定) + 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132' + } + + def start(self): + """请求获取公众号的文章接口""" + + start = True + offset = self.offset + while start: + api = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={0}&f=json&offset={1}' \ + '&count=10&is_ok=1&scene=124&uin=777&key=777&pass_ticket={2}&wxtoken=&appmsg_token' \ + '={3}&x5=1&f=json'.format(self.biz, offset, self.pass_ticket, self.msg_token) + + resp = requests.get(api, headers=self.headers, verify=False).json() + ret, status = resp.get('ret'), resp.get('errmsg') # 状态信息 + if 0 == ret or 'ok' == status: + + general_msg_list = resp['general_msg_list'] + msg_list = json.loads(general_msg_list)['list'] # 获取文章列表 + for msg in msg_list: + comm_msg_info = msg['comm_msg_info'] # 该数据是本次推送多篇文章公共的 + msg_id = comm_msg_info['id'] # 文章id + if msg_id == self.last_msg_id: + print("最新一条啦{}".format(self.last_msg_id)) + start = False + break + post_time = datetime.fromtimestamp(comm_msg_info['datetime']) # 发布时间 + msg_type = comm_msg_info['type'] # 文章类型 + + if 49 == msg_type: + # 图文消息 + app_msg_ext_info = msg.get('app_msg_ext_info') # article原数据 + if app_msg_ext_info: + # 本次推送的首条文章 + self._parse_articles(app_msg_ext_info, msg_id, post_time, msg_type) + # 本次推送的其余文章 + multi_app_msg_item_list = app_msg_ext_info.get('multi_app_msg_item_list') + if multi_app_msg_item_list: + for item in multi_app_msg_item_list: + msg_id = item['fileid'] # 文章id + if msg_id or not isinstance(msg_id, int): + msg_id = int(time.time()) # 设置唯一id,解决部分文章id=0出现唯一索引冲突的情况 + self._parse_articles(item, msg_id, post_time, msg_type) + + # 0：结束；1：继续 + can_msg_continue = resp.get('can_msg_continue') + if not can_msg_continue: + print('Break , Current offset : %d' % offset) + break + offset = resp.get('next_offset') # 下一次请求偏移量 + print('Next offset : %d' % offset) + + @staticmethod + def crawl_article_content(content_url): + """抓取文章内容 + :param content_url: 文章地址 + """ + + try: + html = requests.get(content_url, verify=False).text + except: + print(content_url) + pass + else: + bs = BeautifulSoup(html, 'html.parser') + js_content = bs.find(id='js_content') + if js_content: + p_list = js_content.find_all('p') + content_list = list(map(lambda p: p.text, filter(lambda p: p.text != '', p_list))) + content = ''.join(content_list) + return content + + def _parse_articles(self, info, msg_id, post_time, msg_type): + """解析嵌套文章数据并保存入库""" + + title = info.get('title') # 标题 + cover = info.get('cover') # 封面图 + author = info.get('author') # 作者 + digest = info.get('digest') # 关键字 + source_url = info.get('source_url') # 原文地址 + content_url = info.get('content_url') # 微信地址 + + content_url = content_url.replace('amp;', '').replace('#wechat_redirect', '').replace('http', 'https') + content = self.crawl_article_content(content_url) + article_id = random.randint(1, 1000) + + json_data = {"articleId": article_id, "author": author, "content": content, + "contentURL": content_url, "cover": cover, "digest": digest, + "mpsId": self.mps_id, "msgId": msg_id, "postTime": post_time, + "sourceURL": source_url, "title": title, "msgType": msg_type} + + time.sleep(random.randint(2, 3)) + self._parse_article_detail(content_url, article_id) + + def _parse_article_detail(self, content_url, article_id): + """从文章页提取相关参数用于获取评论,article_id是已保存的文章id""" + + try: + resp = requests.get(content_url, headers=self.headers, verify=False) + except Exception: + print('获取评论失败 {} {}'.format(article_id, content_url)) + else: + html = resp.text + str_comment = re.search(r'var comment_id = "(.*)" \|\| "(.*)" \* 1;', html) + str_msg = re.search(r"var appmsgid = '' \|\| '(.*)'\|\|", html) + str_token = re.search(r'window.appmsg_token = "(.*)";', html) + + if str_comment and str_msg and str_token: + comment_id = str_comment.group(1) # 评论id(固定) + app_msg_id = str_msg.group(1) # 票据id(非固定) + appmsg_token = str_token.group(1) # 票据token(非固定) + + # 缺一不可 + if appmsg_token and app_msg_id and comment_id: + print('Crawl article {} comments'.format(article_id)) + self._crawl_comments(app_msg_id, comment_id, appmsg_token, article_id) + + def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id): + """抓取文章的评论""" + + api = 'https://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&scene=0&__biz={0}' \ + '&appmsgid={1}&idx=1&comment_id={2}&offset=0&limit=100&uin=777&key=777' \ + '&pass_ticket={3}&wxtoken=777&devicetype=android-26&clientversion=26060739' \ + '&appmsg_token={4}&x5=1&f=json'.format(self.biz, app_msg_id, comment_id, + self.pass_ticket, appmsg_token) + try: + resp = requests.get(api, headers=self.headers, verify=False).json() + except: + print('Article {} no Comment'.format(article_id)) + else: + ret, status = resp['base_resp']['ret'], resp['base_resp']['errmsg'] + if ret == 0 or status == 'ok': + elected_comment = resp['elected_comment'] + for comment in elected_comment: + nick_name = comment.get('nick_name') # 昵称 + logo_url = comment.get('logo_url') # 头像 + comment_time = datetime.fromtimestamp(comment.get('create_time')) # 评论时间 + content = comment.get('content') # 评论内容 + content_id = comment.get('content_id') # id + like_num = comment.get('like_num') # 点赞数 + + reply_list = comment.get('reply')['reply_list'] # 回复数据 + reply_content = reply_like_num = reply_create_time = None + if reply_list: + first_reply = reply_list[0] + reply_content = first_reply.get('content') + reply_like_num = first_reply.get('reply_like_num') + reply_create_time = datetime.fromtimestamp(first_reply.get('create_time')) + + +if __name__ == '__main__': + _id = 18 + biz = 'MzA5MzUwNjUzMA==' + app_msg_token = '1002_7GPI%2FMGbXTfm7IDAroWcT8LW2fDl8wk-CBMFLc9EoS3kEatLJPVvSGB1XT73TFnO5cQScD53A_CnP95b' + # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + pass_ticket = '3ZyE1mdoIe8iz1iG1sKLfjFH3Hv6%252BVH0tFnJpWc6gh8mQCMDP%252BTgONnTUpLCUsqe' + wap_sid2 = 'CO3YwOQHElx6U2RGYmVjSmc4QWx6WENNdF9CbmlQdzlpU244ZVRUdmhVSWNVRXVXT3Q2bFlrNmdHdHNsemVsN3RpWW42VTJKVFZiY1phbDlnTVc0Q09DbEdrMmdCT29EQUFBfjCoqYflBTgNQAE=' + cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + wxMps.start() From b1c3461da2f46ad687f7ad13464d955201dddcc0 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 6 Apr 2019 19:20:51 +0800 Subject: [PATCH 027/148] update together spider --- scrapys/together/together/spiders/yiqi.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapys/together/together/spiders/yiqi.py b/scrapys/together/together/spiders/yiqi.py index 0ad3e69..c2685d1 100644 --- a/scrapys/together/together/spiders/yiqi.py +++ b/scrapys/together/together/spiders/yiqi.py @@ -14,13 +14,12 @@ def __init__(self, name=None, **kwargs): super().__init__(name, **kwargs) self.getUserById = 'http://api.wondertech.com.cn/user/v2/users/getById' # self.likeUser = 'http://api.wondertech.com.cn/fmvoice/v1/voice/vlike' - self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiwiZXhw' \ - 'IjoxNTQ4OTAxMDM4LCJ1c2VySWQiOjU4NjQzfQ.-N6EqLyAU_NvXqcNTT' \ - 'awkpfY_6lidC_XjvYJAnHr-AfBTCz2kT1-Sht_sTDZZOB_lDX6bruHTTJ8SzfZyX1JkA' + self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiwiZXhwIjoxNTU3MTIxMjQxLCJ1c2VySWQiOjU4NjQzfQ.lLvZd0L-ged9WdbQmj9ATS10l_GcyPLIR6P2-THjeZ9VA-iifxpfGBQAFtHPctN0aDHcyFWixiWVb5dzUn7JHw' + self.formData = {'token': self.token} def start_requests(self): - for user_id in range(307478, 320000): + for user_id in range(309124, 350000): self.formData['id'] = str(user_id) yield FormRequest(self.getUserById, formdata=self.formData, callback=self.parse, meta={'uid': user_id}) From 8a75fa02ec7caac9057d14bd257942cac5f652fb Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 6 Apr 2019 19:41:54 +0800 Subject: [PATCH 028/148] update jianshu spider --- scrapys/together/together/spiders/yiqi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapys/together/together/spiders/yiqi.py b/scrapys/together/together/spiders/yiqi.py index c2685d1..89432ce 100644 --- a/scrapys/together/together/spiders/yiqi.py +++ b/scrapys/together/together/spiders/yiqi.py @@ -19,7 +19,7 @@ def __init__(self, name=None, **kwargs): self.formData = {'token': self.token} def start_requests(self): - for user_id in range(309124, 350000): + for user_id in range(309130, 350000): self.formData['id'] = str(user_id) yield FormRequest(self.getUserById, formdata=self.formData, callback=self.parse, meta={'uid': user_id}) From e1060387488dea520ffe3456a20859c4da108a21 Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 18 Apr 2019 15:27:09 +0800 Subject: [PATCH 029/148] update robot --- spider/bihu/robot.py | 7 +- spider/jianshu/robot.py | 250 ++++++++++++++++++++++++---------------- 2 files changed, 155 insertions(+), 102 deletions(-) diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py index e641fda..4b968eb 100644 --- a/spider/bihu/robot.py +++ b/spider/bihu/robot.py @@ -17,11 +17,12 @@ def sleep(min_seconds=3, max_seconds=10): # 无头模式 options = Options() -options.add_argument('window-size=1366x728') -options.add_argument('--disable-gpu') +options.add_argument('--headless') +options.add_argument('log-level=3') options.add_argument('--no-sandbox') +options.add_argument('--disable-gpu') +options.add_argument('window-size=1366x728') options.add_argument('--disable-dev-shm-usage') -options.add_argument('--headless') driver = webdriver.Chrome(chrome_options=options) # 登录币乎 diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 5fbdcff..84a7be9 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,103 +1,155 @@ -import os +import random import sys -import requests -from datetime import datetime +import time -from bs4 import BeautifulSoup -from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, BOOLEAN, create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +import requests +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support.wait import WebDriverWait argv = sys.argv -if len(argv) > 1: - # python3 robot.py jianshu - os.system("nohup python3 {} >> /dev/null 2>&1 &".format(argv[0])) -else: - # python3 robot.py - # 基类 - base = declarative_base() - - - def get_now(): - return datetime.now() - - - class Article(base): - __tablename__ = "tb_jianshu_article" - - id = Column(Integer, primary_key=True, autoincrement=True) - article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") - ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") - ic_like = Column(Integer, nullable=False, comment="喜欢人次") - ic_reward = Column(Integer, nullable=False, comment="赞赏人次") - comment_num = Column(Integer, nullable=False, comment="评论人次") - publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") - processed = Column(BOOLEAN, nullable=False, default=False, comment="处理状态。True:已处理;False:未处理") - update_time = Column(TIMESTAMP, nullable=False, comment="更新时间") - create_time = Column(TIMESTAMP, nullable=False, comment="创建时间") - - - class Info(base): - __tablename__ = "tb_jianshu_info" - - id = Column(Integer, primary_key=True, autoincrement=True) - cookie = Column(String(3000), nullable=False) - expired = Column(BOOLEAN, nullable=False) - - - conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" - engine = create_engine(conn, encoding="UTF-8", echo=False) - base.metadata.create_all(engine) - - Session = sessionmaker(bind=engine) - session = Session() - - domain = "https://www.jianshu.com" - note_like = "https://www.jianshu.com/notes/{}/like" - headers = { - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" - } - - - def get_info(): - return session.query(Info).first() - - - def update_info(): - info = session.query(Info).first() - info.expired = True - session.add(info) - session.commit() - - - def get_article(): - return session.query(Article).filter_by(processed=False).all() - - - def update_article(article_id): - article = session.query(Article).get(article_id) - article.processed = True - session.add(article) - session.commit() - - - def may_like(): - for article in get_article(): - info = get_info() - headers["cookie"] = info.cookie - article_detail = domain + article.article_url_suffix - resp = requests.get(article_detail, headers=headers) - if resp.status_code == 200: - bs = BeautifulSoup(resp.text, "html.parser") - meta = bs.find("meta", {"name": "csrf-token"}) - note = bs.find("div", {"data-vcomp": "recommended-notes"}) - - note_id = note["data-note-id"] - headers["x-csrf-token"] = meta["content"] - resp = requests.post(note_like.format(note_id), headers) - - if not resp.status_code == 200: - update_info() - - - may_like() +argv_length = len(argv) + + +def sleep(min_seconds=3, max_seconds=10): + """随机休眠""" + time.sleep(random.randint(min_seconds, max_seconds)) + + +# 无头模式 +options = Options() +options.add_argument('--headless') +options.add_argument('log-level=3') +options.add_argument('--no-sandbox') +options.add_argument('--disable-gpu') +options.add_argument('window-size=1366x728') +options.add_argument('--disable-dev-shm-usage') +driver = webdriver.Chrome(chrome_options=options) + +username = argv[1] if argv_length > 1 else input("请输入登录账号\n") +password = argv[2] if argv_length > 2 else input("请输入登录密码\n") +role = int(argv[3]) if argv_length > 3 else input("请输入Role\n") + +# 登录简书 +jianshu = "https://www.jianshu.com" +driver.get("{}/sign_in".format(jianshu)) +driver.maximize_window() + +if role == 0: + # QQ登录 + driver.find_element_by_id("qq").click() + sleep() + # 切换到最新打开的窗口 + driver.close() + window_handles = driver.window_handles + driver.switch_to.window(window_handles[-1]) + # 切换iframe + i_frame = WebDriverWait(driver, 3).until(lambda d: d.find_element_by_id('ptlogin_iframe')) + driver.switch_to.frame(i_frame) + driver.find_element_by_id("switcher_plogin").click(), sleep() + driver.find_element_by_id("u").send_keys(username) + driver.find_element_by_id("p").send_keys(password) + driver.find_element_by_id("login_button").click() +elif role == 1: + # 豆瓣登录 + driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() + driver.find_element_by_class_name("douban").click() + sleep() + # 切换到最新打开的窗口 + driver.close() + window_handles = driver.window_handles + driver.switch_to.window(window_handles[-1]) + driver.find_element_by_id("inp-alias").send_keys(username) + driver.find_element_by_id("inp-pwd").send_keys(password) + driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() +sleep() + +page_title = driver.title +print("当前：{}".format(page_title)) + +# 1.写文（一言） +driver.get("{}/writer#/".format(jianshu)), sleep() +driver.find_element_by_class_name("_1GsW5").click(), sleep() + +driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) +title_prefix = "每日一言：" if role == 0 else "一言美句：" +driver.find_element_by_class_name("_24i7u").send_keys(title_prefix) + +i = 0 +content = "" +while len(content) < 500: + resp = requests.get("https://v1.hitokoto.cn/").json() + hitokoto = resp.get("hitokoto") + if hitokoto: + i += 1 + content += hitokoto + driver.find_element_by_id("arthur-editor").send_keys(str(i) + ". " + hitokoto) + if len(content) < 500: + driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) + driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) + +# 发布文章 +driver.find_element_by_class_name("tGbI7").click(), sleep(10, 15) +if role == 1: + # 点个喜欢 + driver.find_element_by_class_name("_2-ZiM").click(), sleep() + driver.find_element_by_xpath("/html/body/div[1]/div[3]/ul/li[4]/a/i").click(), sleep() +if role == 0: + # 取消发布 + # driver.get("{}/writer#/".format(jianshu)), sleep() + # driver.find_element_by_css_selector('a[data-action="privatize"]').click() + pass +sleep() + +# 2.阅读(PC无效) +# driver.get(jianshu) +# sleep() +# content_elements = driver.find_elements_by_class_name("content") +# for content_element in content_elements: +# article_element = content_element.find_element_by_class_name("title") +# article_element.click(), sleep(3, 5) +# window_handles = driver.window_handles +# driver.switch_to.window(window_handles[-1]) +# driver.execute_script("window.scrollTo(0,document.body.scrollHeight)"), sleep(1, 3) +# driver.execute_script("window.scrollTo(document.body.scrollHeight,0)"), sleep(1, 3) +# driver.close() +# window_handles = driver.window_handles +# driver.switch_to.window(window_handles[0]) +# sleep() + +# 3.点赞 +# if role == 0: +# jianshu_u = "{}/u/".format(jianshu) +# role_0 = [] +# elif role == 1: +# driver.get("{}/users/68a80b0c87a5/following".format(jianshu)) +# user_list_element = driver.find_element_by_class_name("user-list") +# user_elements = user_list_element.find_elements_by_class_name("avatar") +# for user_element in user_elements: +# user_element.click(), sleep() +# article_list_element = driver.find_element_by_class_name("note-list") +# top_article_time_element = article_list_element.find_element_by_class_name("time") +# if True: +# article_list_element.find_element_by_class_name("wrap-img").click(), sleep() +# window_handles = driver.window_handles +# driver.switch_to.window(window_handles[-1]) +# driver.find_element_by_css_selector("class='iconfont ic-like'").click() +# driver.close() +# sleep() + +# 4.评论 +jianshu_p = "{}/p/".format(jianshu) +if role == 0: + role_0 = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, role_0)), sleep() + driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click() +elif role == 1: + role_1 = "be27870bdba9" + driver.get("{}{}".format(jianshu_p, role_1)), sleep() + driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) + driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38969657').click() +sleep() From a4857493a36cc300726a96d34fcd0efc158dfe8d Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 7 May 2019 23:11:45 +0800 Subject: [PATCH 030/148] update jianshu robot --- spider/jianshu/robot.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 84a7be9..3669300 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -3,6 +3,7 @@ import time import requests +from datetime import datetime from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys @@ -51,7 +52,7 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id("u").send_keys(username) driver.find_element_by_id("p").send_keys(password) driver.find_element_by_id("login_button").click() -elif role == 1: +else: # 豆瓣登录 driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() driver.find_element_by_class_name("douban").click() @@ -70,10 +71,12 @@ def sleep(min_seconds=3, max_seconds=10): # 1.写文（一言） driver.get("{}/writer#/".format(jianshu)), sleep() -driver.find_element_by_class_name("_1GsW5").click(), sleep() +driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) -title_prefix = "每日一言：" if role == 0 else "一言美句：" +title_prefix = "一言美句：" if role == 1 else "每日一言：" +now_hour = datetime.now().hour +title_prefix = "早晨! " if now_hour < 18 else "晚安! " + title_prefix driver.find_element_by_class_name("_24i7u").send_keys(title_prefix) i = 0 @@ -90,16 +93,15 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) # 发布文章 -driver.find_element_by_class_name("tGbI7").click(), sleep(10, 15) +driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click(), sleep(10, 15) if role == 1: # 点个喜欢 driver.find_element_by_class_name("_2-ZiM").click(), sleep() driver.find_element_by_xpath("/html/body/div[1]/div[3]/ul/li[4]/a/i").click(), sleep() -if role == 0: +else: # 取消发布 - # driver.get("{}/writer#/".format(jianshu)), sleep() - # driver.find_element_by_css_selector('a[data-action="privatize"]').click() - pass + driver.get("{}/writer#/".format(jianshu)), sleep() + driver.find_element_by_css_selector("li[class='tGbI7 cztJE']").click() sleep() # 2.阅读(PC无效) @@ -140,16 +142,16 @@ def sleep(min_seconds=3, max_seconds=10): # 4.评论 jianshu_p = "{}/p/".format(jianshu) -if role == 0: - role_0 = "ac02c56c0865" - driver.get("{}{}".format(jianshu_p, role_0)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click() -elif role == 1: +if role == 1: role_1 = "be27870bdba9" driver.get("{}{}".format(jianshu_p, role_1)), sleep() driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click() +else: + role_0 = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, role_0)), sleep() + driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click() sleep() From 9f68d93b4a0095a883e8a17ea6bfa0f9a67e3258 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 7 May 2019 23:14:47 +0800 Subject: [PATCH 031/148] update jianshu robot --- spider/jianshu/robot.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 3669300..164d341 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -69,7 +69,7 @@ def sleep(min_seconds=3, max_seconds=10): page_title = driver.title print("当前：{}".format(page_title)) -# 1.写文（一言） +# 1.写文(一言Hitokoto) driver.get("{}/writer#/".format(jianshu)), sleep() driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() @@ -93,15 +93,7 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) # 发布文章 -driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click(), sleep(10, 15) -if role == 1: - # 点个喜欢 - driver.find_element_by_class_name("_2-ZiM").click(), sleep() - driver.find_element_by_xpath("/html/body/div[1]/div[3]/ul/li[4]/a/i").click(), sleep() -else: - # 取消发布 - driver.get("{}/writer#/".format(jianshu)), sleep() - driver.find_element_by_css_selector("li[class='tGbI7 cztJE']").click() +driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() # 2.阅读(PC无效) @@ -120,7 +112,7 @@ def sleep(min_seconds=3, max_seconds=10): # driver.switch_to.window(window_handles[0]) # sleep() -# 3.点赞 +# 3.点赞(手动操作) # if role == 0: # jianshu_u = "{}/u/".format(jianshu) # role_0 = [] From e35c163ca6a0bda8b9a298c16248837615d84883 Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 8 May 2019 09:10:23 +0800 Subject: [PATCH 032/148] update robot --- spider/jianshu/robot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 164d341..b885cde 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -76,12 +76,12 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) title_prefix = "一言美句：" if role == 1 else "每日一言：" now_hour = datetime.now().hour -title_prefix = "早晨! " if now_hour < 18 else "晚安! " + title_prefix +title_prefix = "早晨！" if now_hour < 18 else "晚安！" + title_prefix driver.find_element_by_class_name("_24i7u").send_keys(title_prefix) i = 0 content = "" -while len(content) < 500: +while len(content) < 1000: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") if hitokoto: From 45f2f32ce7dbdf4c24470d5262bc206d3903a935 Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 8 May 2019 09:10:45 +0800 Subject: [PATCH 033/148] update robot --- spider/jianshu/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index b885cde..3b58a35 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -88,7 +88,7 @@ def sleep(min_seconds=3, max_seconds=10): i += 1 content += hitokoto driver.find_element_by_id("arthur-editor").send_keys(str(i) + ". " + hitokoto) - if len(content) < 500: + if len(content) < 1000: driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) From 71e7148ed658cae0e0b986bfee8779db41174edd Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 8 May 2019 09:28:04 +0800 Subject: [PATCH 034/148] update robot --- spider/jianshu/robot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 3b58a35..a8b6b44 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -74,10 +74,9 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) -title_prefix = "一言美句：" if role == 1 else "每日一言：" -now_hour = datetime.now().hour -title_prefix = "早晨！" if now_hour < 18 else "晚安！" + title_prefix -driver.find_element_by_class_name("_24i7u").send_keys(title_prefix) +title_prefix = "早晨" if datetime.now().hour < 18 else "晚安" +title_prefix2 = "一言美句" if role == 1 else "每日一言" +driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) i = 0 content = "" From ea6358920f8e99c52858b7f51018b7872ad320fc Mon Sep 17 00:00:00 2001 From: happyJared Date: Wed, 8 May 2019 09:30:10 +0800 Subject: [PATCH 035/148] update robot --- spider/jianshu/robot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index a8b6b44..f15965e 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -78,16 +78,16 @@ def sleep(min_seconds=3, max_seconds=10): title_prefix2 = "一言美句" if role == 1 else "每日一言" driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) -i = 0 content = "" -while len(content) < 1000: +i, content_length = 0, random.randint(800, 1500) +while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") if hitokoto: i += 1 content += hitokoto driver.find_element_by_id("arthur-editor").send_keys(str(i) + ". " + hitokoto) - if len(content) < 1000: + if len(content) < content_length: driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) From 2db717a13e63b09b85384fccb70483532d39254c Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 9 May 2019 10:26:20 +0800 Subject: [PATCH 036/148] new mps --- wechat/wx_mps.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 1c7d36f..f02691a 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -16,18 +16,18 @@ elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# def load(): -# for i in range(21, 22): -# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ -# "tb_mps where id = {} and show = True".format(i) -# result = postgres.fetch_all(sql) -# for r in result: -# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ -# r[3], r[4], r[5] -# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( -# r_pass_ticket, r_wap_sid2) -# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) -# mps.start() +def load(): + for i in range(24, 25): + sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ + "tb_mps where id = {} and show = True".format(i) + result = postgres.fetch_all(sql) + for r in result: + r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ + r[3], r[4], r[5] + r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( + r_pass_ticket, r_wap_sid2) + mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) + mps.start() class WxMps(object): @@ -248,14 +248,14 @@ def _save_article_comment(): if __name__ == '__main__': - _id = 18 - biz = 'MjM5NjAxOTU4MA==' - app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # 开始爬取文章及评论 - wxMps.start() - # load() + # _id = 18 + # biz = 'MjM5NjAxOTU4MA==' + # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # # 开始爬取文章及评论 + # wxMps.start() + load() From 75f3d3f3afce069167ec18e0a6eb665135370c29 Mon Sep 17 00:00:00 2001 From: happyJared Date: Thu, 9 May 2019 10:36:06 +0800 Subject: [PATCH 037/148] restore --- wechat/wx_mps.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index f02691a..90f3697 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -16,18 +16,18 @@ elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -def load(): - for i in range(24, 25): - sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ - "tb_mps where id = {} and show = True".format(i) - result = postgres.fetch_all(sql) - for r in result: - r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ - r[3], r[4], r[5] - r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( - r_pass_ticket, r_wap_sid2) - mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) - mps.start() +# def load(): +# for i in range(24, 25): +# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ +# "tb_mps where id = {} and show = True".format(i) +# result = postgres.fetch_all(sql) +# for r in result: +# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ +# r[3], r[4], r[5] +# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( +# r_pass_ticket, r_wap_sid2) +# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) +# mps.start() class WxMps(object): @@ -248,14 +248,14 @@ def _save_article_comment(): if __name__ == '__main__': - # _id = 18 - # biz = 'MjM5NjAxOTU4MA==' - # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # # 开始爬取文章及评论 - # wxMps.start() - load() + _id = 18 + biz = 'MjM5NjAxOTU4MA==' + app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # 开始爬取文章及评论 + wxMps.start() + # load() From 255b833e50035d0dfad51c3d4a9f41a79c544903 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 12 May 2019 20:53:48 +0800 Subject: [PATCH 038/148] update jianshu robot --- spider/jianshu/robot.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index f15965e..3f08717 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -74,7 +74,8 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) -title_prefix = "早晨" if datetime.now().hour < 18 else "晚安" +is_day = datetime.now().hour < 18 +title_prefix = "早晨" if is_day < 18 else "晚安" title_prefix2 = "一言美句" if role == 1 else "每日一言" driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) @@ -132,17 +133,18 @@ def sleep(min_seconds=3, max_seconds=10): # sleep() # 4.评论 -jianshu_p = "{}/p/".format(jianshu) -if role == 1: - role_1 = "be27870bdba9" - driver.get("{}{}".format(jianshu_p, role_1)), sleep() - driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) - driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) - driver.find_element_by_id('like-button-38969657').click() -else: - role_0 = "ac02c56c0865" - driver.get("{}{}".format(jianshu_p, role_0)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click() -sleep() +if is_day: + jianshu_p = "{}/p/".format(jianshu) + if role == 1: + role_1 = "be27870bdba9" + driver.get("{}{}".format(jianshu_p, role_1)), sleep() + driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) + driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38969657').click() + else: + role_0 = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, role_0)), sleep() + driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click() + sleep() From 077a10d43a9cf5c6813321551580a523c98a05ca Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 12 May 2019 21:44:41 +0800 Subject: [PATCH 039/148] update jianshu robot --- spider/jianshu/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 3f08717..fae9dd2 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -75,7 +75,7 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) is_day = datetime.now().hour < 18 -title_prefix = "早晨" if is_day < 18 else "晚安" +title_prefix = "早晨" if is_day else "晚安" title_prefix2 = "一言美句" if role == 1 else "每日一言" driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) From a44e8a8e03a5f47981014a14fa40ed847cc3ddc8 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 14 May 2019 23:32:35 +0800 Subject: [PATCH 040/148] update jianshu robot --- spider/jianshu/robot.py | 106 +++++++++++----------------------------- 1 file changed, 28 insertions(+), 78 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index fae9dd2..374f126 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,13 +1,12 @@ import random import sys import time +from datetime import datetime import requests -from datetime import datetime from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys -from selenium.webdriver.support.wait import WebDriverWait argv = sys.argv argv_length = len(argv) @@ -37,114 +36,65 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}/sign_in".format(jianshu)) driver.maximize_window() -if role == 0: - # QQ登录 - driver.find_element_by_id("qq").click() - sleep() - # 切换到最新打开的窗口 - driver.close() - window_handles = driver.window_handles - driver.switch_to.window(window_handles[-1]) - # 切换iframe - i_frame = WebDriverWait(driver, 3).until(lambda d: d.find_element_by_id('ptlogin_iframe')) - driver.switch_to.frame(i_frame) - driver.find_element_by_id("switcher_plogin").click(), sleep() - driver.find_element_by_id("u").send_keys(username) - driver.find_element_by_id("p").send_keys(password) - driver.find_element_by_id("login_button").click() -else: - # 豆瓣登录 - driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() - driver.find_element_by_class_name("douban").click() - sleep() - # 切换到最新打开的窗口 - driver.close() - window_handles = driver.window_handles - driver.switch_to.window(window_handles[-1]) - driver.find_element_by_id("inp-alias").send_keys(username) - driver.find_element_by_id("inp-pwd").send_keys(password) - driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() +# 豆瓣登录 +driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() +driver.find_element_by_class_name("douban").click() +sleep() +# 切换到最新打开的窗口 +driver.close() +window_handles = driver.window_handles +driver.switch_to.window(window_handles[-1]) +driver.find_element_by_id("inp-alias").send_keys(username) +driver.find_element_by_id("inp-pwd").send_keys(password) +driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() sleep() page_title = driver.title print("当前：{}".format(page_title)) -# 1.写文(一言Hitokoto) +# 1.写文 driver.get("{}/writer#/".format(jianshu)), sleep() driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) is_day = datetime.now().hour < 18 title_prefix = "早晨" if is_day else "晚安" -title_prefix2 = "一言美句" if role == 1 else "每日一言" +title_prefix2 = "每日一言" if role == 0 else "一言美句" driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) content = "" -i, content_length = 0, random.randint(800, 1500) +count, content_length = 0, random.randint(1000, 1500) while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") if hitokoto: - i += 1 + count += 1 content += hitokoto - driver.find_element_by_id("arthur-editor").send_keys(str(i) + ". " + hitokoto) + driver.find_element_by_id("arthur-editor").send_keys(str(count) + ". " + hitokoto) if len(content) < content_length: driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) - # 发布文章 driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() -# 2.阅读(PC无效) -# driver.get(jianshu) -# sleep() -# content_elements = driver.find_elements_by_class_name("content") -# for content_element in content_elements: -# article_element = content_element.find_element_by_class_name("title") -# article_element.click(), sleep(3, 5) -# window_handles = driver.window_handles -# driver.switch_to.window(window_handles[-1]) -# driver.execute_script("window.scrollTo(0,document.body.scrollHeight)"), sleep(1, 3) -# driver.execute_script("window.scrollTo(document.body.scrollHeight,0)"), sleep(1, 3) -# driver.close() -# window_handles = driver.window_handles -# driver.switch_to.window(window_handles[0]) -# sleep() - -# 3.点赞(手动操作) -# if role == 0: -# jianshu_u = "{}/u/".format(jianshu) -# role_0 = [] -# elif role == 1: -# driver.get("{}/users/68a80b0c87a5/following".format(jianshu)) -# user_list_element = driver.find_element_by_class_name("user-list") -# user_elements = user_list_element.find_elements_by_class_name("avatar") -# for user_element in user_elements: -# user_element.click(), sleep() -# article_list_element = driver.find_element_by_class_name("note-list") -# top_article_time_element = article_list_element.find_element_by_class_name("time") -# if True: -# article_list_element.find_element_by_class_name("wrap-img").click(), sleep() -# window_handles = driver.window_handles -# driver.switch_to.window(window_handles[-1]) -# driver.find_element_by_css_selector("class='iconfont ic-like'").click() -# driver.close() -# sleep() - -# 4.评论 +# 2.评论 if is_day: jianshu_p = "{}/p/".format(jianshu) - if role == 1: - role_1 = "be27870bdba9" - driver.get("{}{}".format(jianshu_p, role_1)), sleep() + if role == 0: + p_suffix = "be27870bdba9" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click() - else: - role_0 = "ac02c56c0865" - driver.get("{}{}".format(jianshu_p, role_0)), sleep() + elif role == 1: + p_suffix = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) driver.find_element_by_id('like-button-38968576').click() + elif role == 2: + pass + elif role == 3: + pass sleep() From af8e6667fe5b2e9a2f6bd1022f9901faa06cdeee Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 15 May 2019 08:21:12 +0800 Subject: [PATCH 041/148] update jianshu robot --- spider/jianshu/robot.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 374f126..98281a1 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -82,19 +82,26 @@ def sleep(min_seconds=3, max_seconds=10): if is_day: jianshu_p = "{}/p/".format(jianshu) if role == 0: + p_suffix = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + driver.find_element_by_id('like-button-38968576').click() + elif role == 1: p_suffix = "be27870bdba9" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) driver.find_element_by_id('like-button-38969657').click() - elif role == 1: - p_suffix = "ac02c56c0865" + elif role == 2: + p_suffix = "277f1b0a140d" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click() - elif role == 2: - pass + driver.find_element_by_id('like-button-40692862').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40692862').click() elif role == 3: - pass - sleep() + p_suffix = "15eb212788fc" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,2000)"), sleep(1, 2) + driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40693328').click() From e44a18d5683b4a8c7d629bdb72f23940ab21df36 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 15 May 2019 08:36:38 +0800 Subject: [PATCH 042/148] update jianshu robot --- spider/jianshu/robot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 98281a1..76a43ce 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -84,7 +84,7 @@ def sleep(min_seconds=3, max_seconds=10): if role == 0: p_suffix = "ac02c56c0865" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) driver.find_element_by_id('like-button-38968576').click() elif role == 1: From 5002c761911ec0d5fac5e477709cb361201a7751 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 15 May 2019 08:47:27 +0800 Subject: [PATCH 043/148] update jianshu robot --- spider/jianshu/robot.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 76a43ce..393832d 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -105,3 +105,9 @@ def sleep(min_seconds=3, max_seconds=10): driver.execute_script("window.scrollTo(0,2000)"), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click() + +sleep() +driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/comments")) From 49ba95a680ab75a83e32f4c4014fc1adf6a5fbeb Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 19:43:01 +0800 Subject: [PATCH 044/148] update jianshu robot --- .gitignore | 3 ++- spider/jianshu/start.py | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 spider/jianshu/start.py diff --git a/.gitignore b/.gitignore index 62c8935..0b71a72 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea/ \ No newline at end of file +.idea/ +spider/jianshu/start.sh \ No newline at end of file diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py new file mode 100644 index 0000000..9c06fd6 --- /dev/null +++ b/spider/jianshu/start.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# @author : jared +# @date : 2019/5/16 19:33 + +import os +from apscheduler.schedulers.background import * + + +def robot_job(job): + scheduler = BlockingScheduler() + scheduler.add_job(job, 'cron', hour=6) + scheduler.start() + + +def robot(): + os.system('./start.sh') + + +if __name__ == '__main__': + robot_job(robot) From a7a633a1675b17276b46e9e31a8de916ae35ef0e Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 19:50:09 +0800 Subject: [PATCH 045/148] update jianshu robot --- spider/jianshu/start.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index 9c06fd6..ebedf41 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -6,9 +6,9 @@ from apscheduler.schedulers.background import * -def robot_job(job): +def robot_job(job, hour): scheduler = BlockingScheduler() - scheduler.add_job(job, 'cron', hour=6) + scheduler.add_job(job, 'cron', hour=hour) scheduler.start() @@ -17,4 +17,5 @@ def robot(): if __name__ == '__main__': - robot_job(robot) + os.system("nohup python3 start.py 5 >> /dev/null 2>&1 &") + os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") From d279bc4363e563b039e31589467e710837d8d2e4 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 19:52:35 +0800 Subject: [PATCH 046/148] update jianshu robot --- spider/jianshu/robot.py | 14 ++++++++++++++ spider/jianshu/start.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 393832d..6c297b1 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -105,6 +105,20 @@ def sleep(min_seconds=3, max_seconds=10): driver.execute_script("window.scrollTo(0,2000)"), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click() + elif role == 10: + pass + # p_suffix = "ac02c56c0865" + # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click() + elif role == 11: + pass + # p_suffix = "be27870bdba9" + # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + # driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) + # driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) + # driver.find_element_by_id('like-button-38969657').click() sleep() driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index ebedf41..aedd115 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -17,5 +17,5 @@ def robot(): if __name__ == '__main__': - os.system("nohup python3 start.py 5 >> /dev/null 2>&1 &") + os.system("nohup python3 start.py 23 >> /dev/null 2>&1 &") os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") From 77c2810ce979d967bb4b3c55dc5b09aa6eeccc3f Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 22:05:09 +0800 Subject: [PATCH 047/148] update jianshu robot --- spider/jianshu/robot.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 6c297b1..1423dc1 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -105,6 +105,13 @@ def sleep(min_seconds=3, max_seconds=10): driver.execute_script("window.scrollTo(0,2000)"), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click() + elif role == 4: + pass + # p_suffix = "ac02c56c0865" + # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click() elif role == 10: pass # p_suffix = "ac02c56c0865" From 32a1990a5034f313a5fe788f48423c5e130c4280 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 22:06:59 +0800 Subject: [PATCH 048/148] update jianshu robot --- spider/jianshu/start.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index aedd115..30ff078 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -17,5 +17,5 @@ def robot(): if __name__ == '__main__': - os.system("nohup python3 start.py 23 >> /dev/null 2>&1 &") + os.system("nohup python3 start.py 22 >> /dev/null 2>&1 &") os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") From 6954b82c76c1ff36e4380d0382ec215f2e3d4650 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 16 May 2019 22:12:48 +0800 Subject: [PATCH 049/148] update jianshu robot --- spider/jianshu/start.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index 30ff078..cefddcd 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -3,12 +3,14 @@ # @date : 2019/5/16 19:33 import os +import sys + from apscheduler.schedulers.background import * -def robot_job(job, hour): +def robot_job(hour): scheduler = BlockingScheduler() - scheduler.add_job(job, 'cron', hour=hour) + scheduler.add_job(robot, 'cron', hour=hour) scheduler.start() @@ -17,5 +19,7 @@ def robot(): if __name__ == '__main__': - os.system("nohup python3 start.py 22 >> /dev/null 2>&1 &") - os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") + argv = sys.argv + robot_job(argv[1]) + # os.system("nohup python3 start.py 22 >> /dev/null 2>&1 &") + # os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") From 4157e322921374305134002d376de98af1aa1d37 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 17 May 2019 20:21:13 +0800 Subject: [PATCH 050/148] update jianshu robot --- spider/jianshu/robot.py | 26 +++++++++++++++++++------- spider/jianshu/start.py | 3 +-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 1423dc1..6da0544 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -50,20 +50,25 @@ def sleep(min_seconds=3, max_seconds=10): sleep() page_title = driver.title -print("当前：{}".format(page_title)) +print("当前{} ：{}".format(role, page_title)) # 1.写文 driver.get("{}/writer#/".format(jianshu)), sleep() driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() -driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) is_day = datetime.now().hour < 18 -title_prefix = "早晨" if is_day else "晚安" -title_prefix2 = "每日一言" if role == 0 else "一言美句" -driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) +if role == 0 or role == 1: + title_prefix = "早晨" if is_day else "晚安" + title_prefix2 = "每日一言" if role == 0 else "一言美句" + driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) + driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) +else: + resp = requests.get("https://v1.hitokoto.cn/").json() + title = resp.get('hitokoto') + driver.find_element_by_class_name("_24i7u").clear() + driver.find_element_by_class_name("_24i7u").send_keys(title) -content = "" -count, content_length = 0, random.randint(1000, 1500) +content, count, content_length = "", 0, random.randint(1000, 1500) while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") @@ -112,6 +117,13 @@ def sleep(min_seconds=3, max_seconds=10): # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) # driver.find_element_by_id('like-button-38968576').click() + elif role == 5: + pass + # p_suffix = "ac02c56c0865" + # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + # driver.find_element_by_id('like-button-38968576').click() elif role == 10: pass # p_suffix = "ac02c56c0865" diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index cefddcd..5b999a7 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -18,8 +18,7 @@ def robot(): os.system('./start.sh') +# nohup python3 start.py 22 >> /dev/null 2>&1 & if __name__ == '__main__': argv = sys.argv robot_job(argv[1]) - # os.system("nohup python3 start.py 22 >> /dev/null 2>&1 &") - # os.system("nohup python3 start.py 6 >> /dev/null 2>&1 &") From 3b9b790726a7db9a594d5d7f23aadaefb12880b9 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 17 May 2019 21:29:48 +0800 Subject: [PATCH 051/148] update jianshu robot --- spider/jianshu/robot.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 6da0544..1c6f54e 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -111,19 +111,17 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click() elif role == 4: - pass - # p_suffix = "ac02c56c0865" - # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click() + p_suffix = "81d5ae85f38a" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3600)"), sleep(1, 2) + driver.find_element_by_id('like-button-40874580').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40874580').click() elif role == 5: - pass - # p_suffix = "ac02c56c0865" - # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click() + p_suffix = "4dca23c2c75a" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) + driver.find_element_by_id('like-button-40874520').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40874520').click() elif role == 10: pass # p_suffix = "ac02c56c0865" From 7edc0c22edd61fa10bad4d8b88dee5257466f624 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 18 May 2019 08:54:30 +0800 Subject: [PATCH 052/148] update jianshu robot --- spider/jianshu/robot.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 1c6f54e..6e22060 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -123,19 +123,17 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id('like-button-40874520').click(), sleep(1, 2) driver.find_element_by_id('like-button-40874520').click() elif role == 10: - pass - # p_suffix = "ac02c56c0865" - # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - # driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) - # driver.find_element_by_id('like-button-38968576').click() + p_suffix = "f9a76934b45f" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,2500)"), sleep(1, 2) + driver.find_element_by_id('like-button-40892474').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40892474').click() elif role == 11: - pass - # p_suffix = "be27870bdba9" - # driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - # driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) - # driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) - # driver.find_element_by_id('like-button-38969657').click() + p_suffix = "8f93afad869f" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3800)"), sleep(1, 2) + driver.find_element_by_id('like-button-40892368').click(), sleep(1, 2) + driver.find_element_by_id('like-button-40892368').click() sleep() driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) From 951e69d15fbf8c113271906bc1e779f7044487e7 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 18 May 2019 10:28:27 +0800 Subject: [PATCH 053/148] update jianshu robot --- spider/jianshu/robot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 6e22060..f90ac88 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -139,4 +139,5 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/comments")) From f3d4d15bc68cb4db6d3df5846c86aab64678573f Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 19 May 2019 08:20:05 +0800 Subject: [PATCH 054/148] update jianshu robot --- spider/jianshu/robot.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index f90ac88..dc2ee42 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -141,3 +141,4 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/comments")) +driver.quit() From 3ae24e715c567ff5c2a53b4536db0f923b125484 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 21 May 2019 16:31:00 +0800 Subject: [PATCH 055/148] update jianshu robot --- spider/jianshu/robot.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index dc2ee42..038742b 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,3 +1,5 @@ +import os +import platform import random import sys import time @@ -11,6 +13,13 @@ argv = sys.argv argv_length = len(argv) +username = argv[1] if argv_length > 1 else input("请输入登录账号\n") +password = argv[2] if argv_length > 2 else input("请输入登录密码\n") +role = int(argv[3]) if argv_length > 3 else input("请输入Role\n") + +if platform.system() == "Linux": + os.system('ps -ef | grep chrome | awk "{print \$2}" | xargs kill -9') + def sleep(min_seconds=3, max_seconds=10): """随机休眠""" @@ -27,10 +36,6 @@ def sleep(min_seconds=3, max_seconds=10): options.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome(chrome_options=options) -username = argv[1] if argv_length > 1 else input("请输入登录账号\n") -password = argv[2] if argv_length > 2 else input("请输入登录密码\n") -role = int(argv[3]) if argv_length > 3 else input("请输入Role\n") - # 登录简书 jianshu = "https://www.jianshu.com" driver.get("{}/sign_in".format(jianshu)) @@ -50,7 +55,7 @@ def sleep(min_seconds=3, max_seconds=10): sleep() page_title = driver.title -print("当前{} ：{}".format(role, page_title)) +print("{} ：{}".format(role, page_title)) # 1.写文 driver.get("{}/writer#/".format(jianshu)), sleep() @@ -68,6 +73,12 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").clear() driver.find_element_by_class_name("_24i7u").send_keys(title) +start = '> 动漫也好、小说也好、网络也好，不论在哪里，我们总会看到有那么一两个句子能穿透你的心，这就是一言。' \ + '或是感动，或是开心，又或是单纯的回忆，与大家分享，这就是一言存在的目的。' +driver.find_element_by_id("arthur-editor").send_keys(start) +driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) +driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) + content, count, content_length = "", 0, random.randint(1000, 1500) while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() @@ -101,37 +112,37 @@ def sleep(min_seconds=3, max_seconds=10): elif role == 2: p_suffix = "277f1b0a140d" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) driver.find_element_by_id('like-button-40692862').click(), sleep(1, 2) driver.find_element_by_id('like-button-40692862').click() elif role == 3: p_suffix = "15eb212788fc" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2000)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,2150)"), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) driver.find_element_by_id('like-button-40693328').click() elif role == 4: p_suffix = "81d5ae85f38a" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3600)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3550)"), sleep(1, 2) driver.find_element_by_id('like-button-40874580').click(), sleep(1, 2) driver.find_element_by_id('like-button-40874580').click() elif role == 5: p_suffix = "4dca23c2c75a" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3550)"), sleep(1, 2) driver.find_element_by_id('like-button-40874520').click(), sleep(1, 2) driver.find_element_by_id('like-button-40874520').click() elif role == 10: p_suffix = "f9a76934b45f" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2500)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,2600)"), sleep(1, 2) driver.find_element_by_id('like-button-40892474').click(), sleep(1, 2) driver.find_element_by_id('like-button-40892474').click() elif role == 11: p_suffix = "8f93afad869f" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3800)"), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3900)"), sleep(1, 2) driver.find_element_by_id('like-button-40892368').click(), sleep(1, 2) driver.find_element_by_id('like-button-40892368').click() From 3949d8f6cceecb6dc0d010cb8017ff1b6d59a214 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 22 May 2019 10:16:46 +0800 Subject: [PATCH 056/148] update jianshu robot --- spider/jianshu/robot.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 038742b..3897880 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -73,23 +73,26 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").clear() driver.find_element_by_class_name("_24i7u").send_keys(title) -start = '> 动漫也好、小说也好、网络也好，不论在哪里，我们总会看到有那么一两个句子能穿透你的心，这就是一言。' \ - '或是感动，或是开心，又或是单纯的回忆，与大家分享，这就是一言存在的目的。' -driver.find_element_by_id("arthur-editor").send_keys(start) -driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) -driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) +editor = "arthur-editor" +start = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' +driver.find_element_by_id(editor).send_keys(start) +driver.find_element_by_id(editor).send_keys(Keys.ENTER) +driver.find_element_by_id(editor).send_keys(Keys.ENTER) -content, count, content_length = "", 0, random.randint(1000, 1500) +content, count, content_length = "", 0, random.randint(700, 1200) while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") if hitokoto: count += 1 content += hitokoto - driver.find_element_by_id("arthur-editor").send_keys(str(count) + ". " + hitokoto) + driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) if len(content) < content_length: - driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) - driver.find_element_by_id("arthur-editor").send_keys(Keys.ENTER) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) + +end = '> 或是开心，或是感动，又或是单纯的回忆，与大家分享美句，这就是一言。' +driver.find_element_by_id(editor).send_keys(end) # 发布文章 driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() From cfc7b246d553c5957e89f39d1838b445a0ac6c95 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 23 May 2019 13:35:03 +0800 Subject: [PATCH 057/148] update jianshu robot --- spider/jianshu/robot.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 3897880..957f7a3 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -73,13 +73,8 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").clear() driver.find_element_by_class_name("_24i7u").send_keys(title) -editor = "arthur-editor" -start = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' -driver.find_element_by_id(editor).send_keys(start) -driver.find_element_by_id(editor).send_keys(Keys.ENTER) -driver.find_element_by_id(editor).send_keys(Keys.ENTER) - -content, count, content_length = "", 0, random.randint(700, 1200) +content, editor = "", "arthur-editor" +count, content_length = 0, random.randint(700, 1200) while len(content) < content_length: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") @@ -91,8 +86,9 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id(editor).send_keys(Keys.ENTER) driver.find_element_by_id(editor).send_keys(Keys.ENTER) -end = '> 或是开心，或是感动，又或是单纯的回忆，与大家分享美句，这就是一言。' +end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' driver.find_element_by_id(editor).send_keys(end) +driver.find_element_by_id(editor).send_keys(Keys.ENTER) # 发布文章 driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() From 8b8a81953bfb64a3be3e75f335922706fdca70c4 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 23 May 2019 23:26:05 +0800 Subject: [PATCH 058/148] update jianshu robot --- spider/jianshu/robot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 957f7a3..50e311d 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -86,9 +86,10 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id(editor).send_keys(Keys.ENTER) driver.find_element_by_id(editor).send_keys(Keys.ENTER) +driver.find_element_by_id(editor).send_keys(Keys.ENTER) +driver.find_element_by_id(editor).send_keys(Keys.ENTER) end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' driver.find_element_by_id(editor).send_keys(end) -driver.find_element_by_id(editor).send_keys(Keys.ENTER) # 发布文章 driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() From 4bcaf03d9380ba51ac0af1ef496ac13702a714b4 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 24 May 2019 09:06:47 +0800 Subject: [PATCH 059/148] update jianshu robot --- spider/jianshu/robot.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 50e311d..034bb92 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -81,7 +81,8 @@ def sleep(min_seconds=3, max_seconds=10): if hitokoto: count += 1 content += hitokoto - driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) + mark = "- " if role % 2 == 0 else str(count) + ". " + driver.find_element_by_id(editor).send_keys(mark + hitokoto) if len(content) < content_length: driver.find_element_by_id(editor).send_keys(Keys.ENTER) driver.find_element_by_id(editor).send_keys(Keys.ENTER) From c7cb214d1b8c3a2c39bb1b29a161125c0a83d171 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 25 May 2019 22:59:03 +0800 Subject: [PATCH 060/148] update jianshu robot --- spider/jianshu/robot.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 034bb92..18f6fb5 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -73,25 +73,20 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_class_name("_24i7u").clear() driver.find_element_by_class_name("_24i7u").send_keys(title) -content, editor = "", "arthur-editor" -count, content_length = 0, random.randint(700, 1200) -while len(content) < content_length: +content, editor, count = "", "arthur-editor", 0 +while len(content) < 600: resp = requests.get("https://v1.hitokoto.cn/").json() hitokoto = resp.get("hitokoto") if hitokoto: count += 1 - content += hitokoto + content += hitokoto.strip() mark = "- " if role % 2 == 0 else str(count) + ". " driver.find_element_by_id(editor).send_keys(mark + hitokoto) - if len(content) < content_length: - driver.find_element_by_id(editor).send_keys(Keys.ENTER) - driver.find_element_by_id(editor).send_keys(Keys.ENTER) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) -driver.find_element_by_id(editor).send_keys(Keys.ENTER) -driver.find_element_by_id(editor).send_keys(Keys.ENTER) end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' driver.find_element_by_id(editor).send_keys(end) -# 发布文章 driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() From 61990e117c1617b53cf807c472d40502404651ef Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 26 May 2019 08:57:19 +0800 Subject: [PATCH 061/148] update jianshu robot --- spider/jianshu/robot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 18f6fb5..ad987ac 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -75,13 +75,13 @@ def sleep(min_seconds=3, max_seconds=10): content, editor, count = "", "arthur-editor", 0 while len(content) < 600: - resp = requests.get("https://v1.hitokoto.cn/").json() + api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) + resp = requests.get("https://{}".format(api)).json() hitokoto = resp.get("hitokoto") if hitokoto: count += 1 content += hitokoto.strip() - mark = "- " if role % 2 == 0 else str(count) + ". " - driver.find_element_by_id(editor).send_keys(mark + hitokoto) + driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) driver.find_element_by_id(editor).send_keys(Keys.ENTER) driver.find_element_by_id(editor).send_keys(Keys.ENTER) From 38f9213080ea0daf32569d7a7a2eda49bca93bc8 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 30 May 2019 23:08:41 +0800 Subject: [PATCH 062/148] update jianshu robot --- scrapys/music163/music163/util/test.py | 185 ------------------------- spider/jianshu/robot.py | 37 ++--- 2 files changed, 20 insertions(+), 202 deletions(-) diff --git a/scrapys/music163/music163/util/test.py b/scrapys/music163/music163/util/test.py index 26a2fde..e69de29 100644 --- a/scrapys/music163/music163/util/test.py +++ b/scrapys/music163/music163/util/test.py @@ -1,185 +0,0 @@ -import json -import random -import re -import time -import urllib3 -from datetime import datetime - -import requests -from bs4 import BeautifulSoup - -urllib3.disable_warnings() - - -class WxMps(object): - """微信公众号文章、评论抓取爬虫""" - - def __init__(self, _mps_id, _biz, _pass_ticket, _app_msg_token, _cookie, last_msg_id=0, _offset=0): - self.offset = _offset - self.mps_id = _mps_id - self.last_msg_id = last_msg_id # 上次抓取的最后消息的id - self.biz = _biz # 公众号标志 - self.msg_token = _app_msg_token # 票据(非固定) - self.pass_ticket = _pass_ticket # 票据(非固定) - self.headers = { - 'Cookie': _cookie, # Cookie(非固定) - 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132' - } - - def start(self): - """请求获取公众号的文章接口""" - - start = True - offset = self.offset - while start: - api = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={0}&f=json&offset={1}' \ - '&count=10&is_ok=1&scene=124&uin=777&key=777&pass_ticket={2}&wxtoken=&appmsg_token' \ - '={3}&x5=1&f=json'.format(self.biz, offset, self.pass_ticket, self.msg_token) - - resp = requests.get(api, headers=self.headers, verify=False).json() - ret, status = resp.get('ret'), resp.get('errmsg') # 状态信息 - if 0 == ret or 'ok' == status: - - general_msg_list = resp['general_msg_list'] - msg_list = json.loads(general_msg_list)['list'] # 获取文章列表 - for msg in msg_list: - comm_msg_info = msg['comm_msg_info'] # 该数据是本次推送多篇文章公共的 - msg_id = comm_msg_info['id'] # 文章id - if msg_id == self.last_msg_id: - print("最新一条啦{}".format(self.last_msg_id)) - start = False - break - post_time = datetime.fromtimestamp(comm_msg_info['datetime']) # 发布时间 - msg_type = comm_msg_info['type'] # 文章类型 - - if 49 == msg_type: - # 图文消息 - app_msg_ext_info = msg.get('app_msg_ext_info') # article原数据 - if app_msg_ext_info: - # 本次推送的首条文章 - self._parse_articles(app_msg_ext_info, msg_id, post_time, msg_type) - # 本次推送的其余文章 - multi_app_msg_item_list = app_msg_ext_info.get('multi_app_msg_item_list') - if multi_app_msg_item_list: - for item in multi_app_msg_item_list: - msg_id = item['fileid'] # 文章id - if msg_id or not isinstance(msg_id, int): - msg_id = int(time.time()) # 设置唯一id,解决部分文章id=0出现唯一索引冲突的情况 - self._parse_articles(item, msg_id, post_time, msg_type) - - # 0：结束；1：继续 - can_msg_continue = resp.get('can_msg_continue') - if not can_msg_continue: - print('Break , Current offset : %d' % offset) - break - offset = resp.get('next_offset') # 下一次请求偏移量 - print('Next offset : %d' % offset) - - @staticmethod - def crawl_article_content(content_url): - """抓取文章内容 - :param content_url: 文章地址 - """ - - try: - html = requests.get(content_url, verify=False).text - except: - print(content_url) - pass - else: - bs = BeautifulSoup(html, 'html.parser') - js_content = bs.find(id='js_content') - if js_content: - p_list = js_content.find_all('p') - content_list = list(map(lambda p: p.text, filter(lambda p: p.text != '', p_list))) - content = ''.join(content_list) - return content - - def _parse_articles(self, info, msg_id, post_time, msg_type): - """解析嵌套文章数据并保存入库""" - - title = info.get('title') # 标题 - cover = info.get('cover') # 封面图 - author = info.get('author') # 作者 - digest = info.get('digest') # 关键字 - source_url = info.get('source_url') # 原文地址 - content_url = info.get('content_url') # 微信地址 - - content_url = content_url.replace('amp;', '').replace('#wechat_redirect', '').replace('http', 'https') - content = self.crawl_article_content(content_url) - article_id = random.randint(1, 1000) - - json_data = {"articleId": article_id, "author": author, "content": content, - "contentURL": content_url, "cover": cover, "digest": digest, - "mpsId": self.mps_id, "msgId": msg_id, "postTime": post_time, - "sourceURL": source_url, "title": title, "msgType": msg_type} - - time.sleep(random.randint(2, 3)) - self._parse_article_detail(content_url, article_id) - - def _parse_article_detail(self, content_url, article_id): - """从文章页提取相关参数用于获取评论,article_id是已保存的文章id""" - - try: - resp = requests.get(content_url, headers=self.headers, verify=False) - except Exception: - print('获取评论失败 {} {}'.format(article_id, content_url)) - else: - html = resp.text - str_comment = re.search(r'var comment_id = "(.*)" \|\| "(.*)" \* 1;', html) - str_msg = re.search(r"var appmsgid = '' \|\| '(.*)'\|\|", html) - str_token = re.search(r'window.appmsg_token = "(.*)";', html) - - if str_comment and str_msg and str_token: - comment_id = str_comment.group(1) # 评论id(固定) - app_msg_id = str_msg.group(1) # 票据id(非固定) - appmsg_token = str_token.group(1) # 票据token(非固定) - - # 缺一不可 - if appmsg_token and app_msg_id and comment_id: - print('Crawl article {} comments'.format(article_id)) - self._crawl_comments(app_msg_id, comment_id, appmsg_token, article_id) - - def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id): - """抓取文章的评论""" - - api = 'https://mp.weixin.qq.com/mp/appmsg_comment?action=getcomment&scene=0&__biz={0}' \ - '&appmsgid={1}&idx=1&comment_id={2}&offset=0&limit=100&uin=777&key=777' \ - '&pass_ticket={3}&wxtoken=777&devicetype=android-26&clientversion=26060739' \ - '&appmsg_token={4}&x5=1&f=json'.format(self.biz, app_msg_id, comment_id, - self.pass_ticket, appmsg_token) - try: - resp = requests.get(api, headers=self.headers, verify=False).json() - except: - print('Article {} no Comment'.format(article_id)) - else: - ret, status = resp['base_resp']['ret'], resp['base_resp']['errmsg'] - if ret == 0 or status == 'ok': - elected_comment = resp['elected_comment'] - for comment in elected_comment: - nick_name = comment.get('nick_name') # 昵称 - logo_url = comment.get('logo_url') # 头像 - comment_time = datetime.fromtimestamp(comment.get('create_time')) # 评论时间 - content = comment.get('content') # 评论内容 - content_id = comment.get('content_id') # id - like_num = comment.get('like_num') # 点赞数 - - reply_list = comment.get('reply')['reply_list'] # 回复数据 - reply_content = reply_like_num = reply_create_time = None - if reply_list: - first_reply = reply_list[0] - reply_content = first_reply.get('content') - reply_like_num = first_reply.get('reply_like_num') - reply_create_time = datetime.fromtimestamp(first_reply.get('create_time')) - - -if __name__ == '__main__': - _id = 18 - biz = 'MzA5MzUwNjUzMA==' - app_msg_token = '1002_7GPI%2FMGbXTfm7IDAroWcT8LW2fDl8wk-CBMFLc9EoS3kEatLJPVvSGB1XT73TFnO5cQScD53A_CnP95b' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '3ZyE1mdoIe8iz1iG1sKLfjFH3Hv6%252BVH0tFnJpWc6gh8mQCMDP%252BTgONnTUpLCUsqe' - wap_sid2 = 'CO3YwOQHElx6U2RGYmVjSmc4QWx6WENNdF9CbmlQdzlpU244ZVRUdmhVSWNVRXVXT3Q2bFlrNmdHdHNsemVsN3RpWW42VTJKVFZiY1phbDlnTVc0Q09DbEdrMmdCT29EQUFBfjCoqYflBTgNQAE=' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - wxMps.start() diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index ad987ac..154c76e 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,4 +1,5 @@ import os +import logging import platform import random import sys @@ -26,6 +27,8 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) +logging.basicConfig(level='INFO', filename='info.log', + format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') # 无头模式 options = Options() options.add_argument('--headless') @@ -55,7 +58,7 @@ def sleep(min_seconds=3, max_seconds=10): sleep() page_title = driver.title -print("{} ：{}".format(role, page_title)) +logging.info("{} ：{}".format(role, page_title)) # 1.写文 driver.get("{}/writer#/".format(jianshu)), sleep() @@ -96,50 +99,50 @@ def sleep(min_seconds=3, max_seconds=10): if role == 0: p_suffix = "ac02c56c0865" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(1, 2) - driver.find_element_by_id('like-button-38968576').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3200)"), sleep(3, 5) + driver.find_element_by_id('like-button-38968576').click(), sleep(3, 5) driver.find_element_by_id('like-button-38968576').click() elif role == 1: p_suffix = "be27870bdba9" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,1400)"), sleep(1, 2) - driver.find_element_by_id('like-button-38969657').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,1400)"), sleep(3, 5) + driver.find_element_by_id('like-button-38969657').click(), sleep(3, 5) driver.find_element_by_id('like-button-38969657').click() elif role == 2: p_suffix = "277f1b0a140d" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3400)"), sleep(1, 2) - driver.find_element_by_id('like-button-40692862').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3400)"), sleep(3, 5) + driver.find_element_by_id('like-button-40692862').click(), sleep(3, 5) driver.find_element_by_id('like-button-40692862').click() elif role == 3: p_suffix = "15eb212788fc" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2150)"), sleep(1, 2) - driver.find_element_by_id('like-button-40693328').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,2150)"), sleep(3, 5) + driver.find_element_by_id('like-button-40693328').click(), sleep(3, 5) driver.find_element_by_id('like-button-40693328').click() elif role == 4: p_suffix = "81d5ae85f38a" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(1, 2) - driver.find_element_by_id('like-button-40874580').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) + driver.find_element_by_id('like-button-40874580').click(), sleep(3, 5) driver.find_element_by_id('like-button-40874580').click() elif role == 5: p_suffix = "4dca23c2c75a" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(1, 2) - driver.find_element_by_id('like-button-40874520').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) + driver.find_element_by_id('like-button-40874520').click(), sleep(3, 5) driver.find_element_by_id('like-button-40874520').click() elif role == 10: p_suffix = "f9a76934b45f" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2600)"), sleep(1, 2) - driver.find_element_by_id('like-button-40892474').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,2600)"), sleep(3, 5) + driver.find_element_by_id('like-button-40892474').click(), sleep(3, 5) driver.find_element_by_id('like-button-40892474').click() elif role == 11: p_suffix = "8f93afad869f" driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3900)"), sleep(1, 2) - driver.find_element_by_id('like-button-40892368').click(), sleep(1, 2) + driver.execute_script("window.scrollTo(0,3900)"), sleep(3, 5) + driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) driver.find_element_by_id('like-button-40892368').click() sleep() From f9e8655c4736938e6ff2f81979e248f34c0fd521 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 30 May 2019 23:13:26 +0800 Subject: [PATCH 063/148] update jianshu robot --- spider/jianshu/robot.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 154c76e..ba3d490 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -93,6 +93,13 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() sleep() +driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) +driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) +sleep() + # 2.评论 if is_day: jianshu_p = "{}/p/".format(jianshu) @@ -145,10 +152,4 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) driver.find_element_by_id('like-button-40892368').click() -sleep() -driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/comments")) driver.quit() From 060bf12531b4d95194836fbae2fa057ab5b98457 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 31 May 2019 16:09:33 +0800 Subject: [PATCH 064/148] update jianshu robot --- spider/jianshu/robot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index ba3d490..6346fc0 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -100,7 +100,14 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) sleep() -# 2.评论 +# 2. 收益 +driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) +elements = driver.find_elements_by_class_name("order") +info = ' ; '.join(map(lambda ele: ele.text, elements[1:])) +logging.info("{} : {}".format(role, info)) +sleep() + +# 3.评论 if is_day: jianshu_p = "{}/p/".format(jianshu) if role == 0: @@ -151,5 +158,4 @@ def sleep(min_seconds=3, max_seconds=10): driver.execute_script("window.scrollTo(0,3900)"), sleep(3, 5) driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) driver.find_element_by_id('like-button-40892368').click() - driver.quit() From 6cbb385c7b8f079adecbf1dd7a3deea0933b610c Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 31 May 2019 22:46:14 +0800 Subject: [PATCH 065/148] update jianshu robot --- spider/jianshu/robot.py | 217 +++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 104 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 6346fc0..d308bc1 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -48,114 +48,123 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() driver.find_element_by_class_name("douban").click() sleep() -# 切换到最新打开的窗口 + +# 切换窗口 driver.close() window_handles = driver.window_handles driver.switch_to.window(window_handles[-1]) driver.find_element_by_id("inp-alias").send_keys(username) driver.find_element_by_id("inp-pwd").send_keys(password) -driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() -sleep() - -page_title = driver.title -logging.info("{} ：{}".format(role, page_title)) -# 1.写文 -driver.get("{}/writer#/".format(jianshu)), sleep() -driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() - -is_day = datetime.now().hour < 18 -if role == 0 or role == 1: - title_prefix = "早晨" if is_day else "晚安" - title_prefix2 = "每日一言" if role == 0 else "一言美句" - driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) - driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) +try: + driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() +except: + logging.info("{} : 登录失败".format(role)) else: - resp = requests.get("https://v1.hitokoto.cn/").json() - title = resp.get('hitokoto') - driver.find_element_by_class_name("_24i7u").clear() - driver.find_element_by_class_name("_24i7u").send_keys(title) - -content, editor, count = "", "arthur-editor", 0 -while len(content) < 600: - api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) - resp = requests.get("https://{}".format(api)).json() - hitokoto = resp.get("hitokoto") - if hitokoto: - count += 1 - content += hitokoto.strip() - driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) - driver.find_element_by_id(editor).send_keys(Keys.ENTER) - driver.find_element_by_id(editor).send_keys(Keys.ENTER) - -end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' -driver.find_element_by_id(editor).send_keys(end) -driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() -sleep() - -driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) -driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) -sleep() - -# 2. 收益 -driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) -elements = driver.find_elements_by_class_name("order") -info = ' ; '.join(map(lambda ele: ele.text, elements[1:])) -logging.info("{} : {}".format(role, info)) -sleep() - -# 3.评论 -if is_day: - jianshu_p = "{}/p/".format(jianshu) - if role == 0: - p_suffix = "ac02c56c0865" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(3, 5) - driver.find_element_by_id('like-button-38968576').click(), sleep(3, 5) - driver.find_element_by_id('like-button-38968576').click() - elif role == 1: - p_suffix = "be27870bdba9" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,1400)"), sleep(3, 5) - driver.find_element_by_id('like-button-38969657').click(), sleep(3, 5) - driver.find_element_by_id('like-button-38969657').click() - elif role == 2: - p_suffix = "277f1b0a140d" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3400)"), sleep(3, 5) - driver.find_element_by_id('like-button-40692862').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40692862').click() - elif role == 3: - p_suffix = "15eb212788fc" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2150)"), sleep(3, 5) - driver.find_element_by_id('like-button-40693328').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40693328').click() - elif role == 4: - p_suffix = "81d5ae85f38a" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) - driver.find_element_by_id('like-button-40874580').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40874580').click() - elif role == 5: - p_suffix = "4dca23c2c75a" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) - driver.find_element_by_id('like-button-40874520').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40874520').click() - elif role == 10: - p_suffix = "f9a76934b45f" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2600)"), sleep(3, 5) - driver.find_element_by_id('like-button-40892474').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40892474').click() - elif role == 11: - p_suffix = "8f93afad869f" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3900)"), sleep(3, 5) - driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40892368').click() -driver.quit() + sleep() + + page_title = driver.title + logging.info("{} ：{}".format(role, page_title)) + + # 1. 收益 + driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) + elements = driver.find_elements_by_class_name("order") + info = ' ; '.join(map(lambda ele: ele.text, elements[1:])) + logging.info("{} : {}".format(role, info)) + sleep() + + # 2.写文 + driver.get("{}/writer#/".format(jianshu)), sleep() + driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() + + is_day = datetime.now().hour < 18 + if role == 0 or role == 1: + title_prefix = "早晨" if is_day else "晚安" + title_prefix2 = "每日一言" if role == 0 else "一言美句" + driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) + driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) + else: + resp = requests.get("https://v1.hitokoto.cn/").json() + title = resp.get('hitokoto') + driver.find_element_by_class_name("_24i7u").clear() + driver.find_element_by_class_name("_24i7u").send_keys(title) + + content, editor, count = "", "arthur-editor", 0 + while len(content) < 600: + api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) + resp = requests.get("https://{}".format(api)).json() + hitokoto = resp.get("hitokoto") + if hitokoto: + count += 1 + content += hitokoto.strip() + driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) + driver.find_element_by_id(editor).send_keys(Keys.ENTER) + + end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' + driver.find_element_by_id(editor).send_keys(end) + driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() + sleep() + + # 3.消息 + driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) + sleep() + + # 4.评论 + if is_day: + jianshu_p = "{}/p/".format(jianshu) + if role == 0: + p_suffix = "ac02c56c0865" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3200)"), sleep(3, 5) + driver.find_element_by_id('like-button-38968576').click(), sleep(3, 5) + driver.find_element_by_id('like-button-38968576').click() + elif role == 1: + p_suffix = "be27870bdba9" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,1400)"), sleep(3, 5) + driver.find_element_by_id('like-button-38969657').click(), sleep(3, 5) + driver.find_element_by_id('like-button-38969657').click() + elif role == 2: + p_suffix = "277f1b0a140d" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3400)"), sleep(3, 5) + driver.find_element_by_id('like-button-40692862').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40692862').click() + elif role == 3: + p_suffix = "15eb212788fc" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,2150)"), sleep(3, 5) + driver.find_element_by_id('like-button-40693328').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40693328').click() + elif role == 4: + p_suffix = "81d5ae85f38a" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) + driver.find_element_by_id('like-button-40874580').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40874580').click() + elif role == 5: + p_suffix = "4dca23c2c75a" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) + driver.find_element_by_id('like-button-40874520').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40874520').click() + elif role == 10: + p_suffix = "f9a76934b45f" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,2600)"), sleep(3, 5) + driver.find_element_by_id('like-button-40892474').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40892474').click() + elif role == 11: + p_suffix = "8f93afad869f" + driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() + driver.execute_script("window.scrollTo(0,3900)"), sleep(3, 5) + driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) + driver.find_element_by_id('like-button-40892368').click() +finally: + driver.close() + driver.quit() From d8be5202a25010783ac11351566462f42a3be17c Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 1 Jun 2019 09:42:09 +0800 Subject: [PATCH 066/148] update jianshu robot --- spider/jianshu/robot.py | 66 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 34 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index d308bc1..c013a91 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,15 +1,13 @@ -import os import logging +import os import platform import random import sys import time from datetime import datetime -import requests from selenium import webdriver from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.keys import Keys argv = sys.argv argv_length = len(argv) @@ -73,38 +71,38 @@ def sleep(min_seconds=3, max_seconds=10): logging.info("{} : {}".format(role, info)) sleep() - # 2.写文 - driver.get("{}/writer#/".format(jianshu)), sleep() - driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() - is_day = datetime.now().hour < 18 - if role == 0 or role == 1: - title_prefix = "早晨" if is_day else "晚安" - title_prefix2 = "每日一言" if role == 0 else "一言美句" - driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) - driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) - else: - resp = requests.get("https://v1.hitokoto.cn/").json() - title = resp.get('hitokoto') - driver.find_element_by_class_name("_24i7u").clear() - driver.find_element_by_class_name("_24i7u").send_keys(title) - - content, editor, count = "", "arthur-editor", 0 - while len(content) < 600: - api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) - resp = requests.get("https://{}".format(api)).json() - hitokoto = resp.get("hitokoto") - if hitokoto: - count += 1 - content += hitokoto.strip() - driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) - driver.find_element_by_id(editor).send_keys(Keys.ENTER) - driver.find_element_by_id(editor).send_keys(Keys.ENTER) - - end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' - driver.find_element_by_id(editor).send_keys(end) - driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() - sleep() + # 2.写文 + # driver.get("{}/writer#/".format(jianshu)), sleep() + # driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() + # + # if role == 0 or role == 1: + # title_prefix = "早晨" if is_day else "晚安" + # title_prefix2 = "每日一言" if role == 0 else "一言美句" + # driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) + # driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) + # else: + # resp = requests.get("https://v1.hitokoto.cn/").json() + # title = resp.get('hitokoto') + # driver.find_element_by_class_name("_24i7u").clear() + # driver.find_element_by_class_name("_24i7u").send_keys(title) + # + # content, editor, count = "", "arthur-editor", 0 + # while len(content) < 600: + # api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) + # resp = requests.get("https://{}".format(api)).json() + # hitokoto = resp.get("hitokoto") + # if hitokoto: + # count += 1 + # content += hitokoto.strip() + # driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) + # driver.find_element_by_id(editor).send_keys(Keys.ENTER) + # driver.find_element_by_id(editor).send_keys(Keys.ENTER) + # + # end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' + # driver.find_element_by_id(editor).send_keys(end) + # driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() + # sleep() # 3.消息 driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) From f5333a75ea6261266a8f34b518519476bb3b9752 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 2 Jun 2019 11:35:23 +0800 Subject: [PATCH 067/148] update jianshu robot --- spider/jianshu/robot.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index c013a91..69e51ce 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -30,11 +30,7 @@ def sleep(min_seconds=3, max_seconds=10): # 无头模式 options = Options() options.add_argument('--headless') -options.add_argument('log-level=3') -options.add_argument('--no-sandbox') -options.add_argument('--disable-gpu') options.add_argument('window-size=1366x728') -options.add_argument('--disable-dev-shm-usage') driver = webdriver.Chrome(chrome_options=options) # 登录简书 From 0a641b91131ddd086b4457dbecd4d18ef5926154 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 2 Jun 2019 11:39:06 +0800 Subject: [PATCH 068/148] update jianshu robot --- spider/jianshu/robot.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 69e51ce..af86c7c 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -30,7 +30,12 @@ def sleep(min_seconds=3, max_seconds=10): # 无头模式 options = Options() options.add_argument('--headless') +options.add_argument('log-level=3') +options.add_argument('--no-sandbox') +options.add_argument('--disable-gpu') options.add_argument('window-size=1366x728') +options.add_argument('--disable-dev-shm-usage') +options.add_argument('blink-settings=imagesEnabled=false') driver = webdriver.Chrome(chrome_options=options) # 登录简书 From bf6c7472a3f8160316269f4b87e06a08b4bb51ae Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 14 Jun 2019 20:00:15 +0800 Subject: [PATCH 069/148] update wx itchat --- wechat/wx_itchat.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wechat/wx_itchat.py b/wechat/wx_itchat.py index 59f83d9..44c74fe 100644 --- a/wechat/wx_itchat.py +++ b/wechat/wx_itchat.py @@ -138,7 +138,7 @@ def _handle_province(self, member_count): for province in list(self.num_of_province.keys()): number = self.num_of_province[province] - if number / member_count < .02 and province != self.unknown_province: + if number / member_count < .01 and province != self.unknown_province: other_province_num = self.num_of_province.get(self.unknown_province) self.num_of_province.__setitem__(self.unknown_province, other_province_num + number) del self.num_of_province[province] @@ -150,6 +150,9 @@ def _plt_gender_bar(self, title): # font size include: ‘xx-small’,x-small’,'small’,'medium’,‘large’,‘x-large’,‘xx-large’ or number # plt.xticks(size='small',rotation=30) + + plt.rcParams["font.sans-serif"] = ["SimHei"] + plt.rcParams["axes.unicode_minus"] = False plt.bar('男', self.male_num, color='yellow') plt.bar('女', self.female_num, color='pink') plt.bar('未知', self.unknown_gender, color='gray') @@ -166,6 +169,8 @@ def _plt_province_pie(self, title): data = np.array(list(self.num_of_province.values())) labels = list(self.num_of_province.keys()) + plt.rcParams["font.sans-serif"] = ["SimHei"] + plt.rcParams["axes.unicode_minus"] = False plt.pie(data, labels=labels, autopct='%.1f%%', ) plt.axis('equal') plt.legend(loc=2, prop={'size': 5.5}) From de947c7fc50111dc496afb63f08bda58a048a65e Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 22 Jun 2019 14:47:53 +0800 Subject: [PATCH 070/148] update jianshu robot --- spider/jianshu/robot.py | 62 +++++++---------------------------------- 1 file changed, 10 insertions(+), 52 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index af86c7c..6406134 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -4,7 +4,6 @@ import random import sys import time -from datetime import datetime from selenium import webdriver from selenium.webdriver.chrome.options import Options @@ -72,7 +71,6 @@ def sleep(min_seconds=3, max_seconds=10): logging.info("{} : {}".format(role, info)) sleep() - is_day = datetime.now().hour < 18 # 2.写文 # driver.get("{}/writer#/".format(jianshu)), sleep() # driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() @@ -114,56 +112,16 @@ def sleep(min_seconds=3, max_seconds=10): sleep() # 4.评论 - if is_day: - jianshu_p = "{}/p/".format(jianshu) - if role == 0: - p_suffix = "ac02c56c0865" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3200)"), sleep(3, 5) - driver.find_element_by_id('like-button-38968576').click(), sleep(3, 5) - driver.find_element_by_id('like-button-38968576').click() - elif role == 1: - p_suffix = "be27870bdba9" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,1400)"), sleep(3, 5) - driver.find_element_by_id('like-button-38969657').click(), sleep(3, 5) - driver.find_element_by_id('like-button-38969657').click() - elif role == 2: - p_suffix = "277f1b0a140d" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3400)"), sleep(3, 5) - driver.find_element_by_id('like-button-40692862').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40692862').click() - elif role == 3: - p_suffix = "15eb212788fc" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2150)"), sleep(3, 5) - driver.find_element_by_id('like-button-40693328').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40693328').click() - elif role == 4: - p_suffix = "81d5ae85f38a" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) - driver.find_element_by_id('like-button-40874580').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40874580').click() - elif role == 5: - p_suffix = "4dca23c2c75a" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3550)"), sleep(3, 5) - driver.find_element_by_id('like-button-40874520').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40874520').click() - elif role == 10: - p_suffix = "f9a76934b45f" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,2600)"), sleep(3, 5) - driver.find_element_by_id('like-button-40892474').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40892474').click() - elif role == 11: - p_suffix = "8f93afad869f" - driver.get("{}{}".format(jianshu_p, p_suffix)), sleep() - driver.execute_script("window.scrollTo(0,3900)"), sleep(3, 5) - driver.find_element_by_id('like-button-40892368').click(), sleep(3, 5) - driver.find_element_by_id('like-button-40892368').click() + jianshu_p = "{}/p/".format(jianshu) + comments = {"ac02c56c0865": "3200,38968576", "be27870bdba9": "1400,38969657", "277f1b0a140d": "1400,40692862", + "15eb212788fc": "2150,40693328", "81d5ae85f38a": "3550,40874580", "4dca23c2c75a": "3550,40874520"} + for comment_id, info in comments.items(): + driver.get("{}{}".format(jianshu_p, comment_id)), sleep() + y_scroll, button_id = info.split(",") + driver.execute_script("window.scrollTo(0,{})".format(y_scroll)), sleep(3, 5) + button_element = 'like-button-{}'.format(button_id) + driver.find_element_by_id(button_element).click(), sleep(3, 5) + driver.find_element_by_id(button_element).click() finally: driver.close() driver.quit() From 8965f2389c9be9a0bc8e1cfed5bd62c83f985282 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 22 Jun 2019 14:49:19 +0800 Subject: [PATCH 071/148] update jianshu robot --- spider/jianshu/robot.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 6406134..aeab0de 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -120,8 +120,11 @@ def sleep(min_seconds=3, max_seconds=10): y_scroll, button_id = info.split(",") driver.execute_script("window.scrollTo(0,{})".format(y_scroll)), sleep(3, 5) button_element = 'like-button-{}'.format(button_id) - driver.find_element_by_id(button_element).click(), sleep(3, 5) - driver.find_element_by_id(button_element).click() + try: + driver.find_element_by_id(button_element).click(), sleep(3, 5) + driver.find_element_by_id(button_element).click() + except: + pass finally: driver.close() driver.quit() From 5f1c687db841abfca289c8e49d91334a3a72d9a3 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 23 Jun 2019 23:06:45 +0800 Subject: [PATCH 072/148] init chelaile --- wechat/chelaile.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 wechat/chelaile.py diff --git a/wechat/chelaile.py b/wechat/chelaile.py new file mode 100644 index 0000000..3994957 --- /dev/null +++ b/wechat/chelaile.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# @author : jared +# @date : 2019/6/22 17:39 + +import time +import json +import itchat +import requests + + +@itchat.msg_register(itchat.content.TEXT) +def text(msg): + receive_text = msg['Text'] + to_user_id = msg['ToUserName'] # 接收人 + from_user_id = msg['FromUserName'] # 发送人 + + while receive_text == "1": + resp = requests.post('https://api.chelaile.net.cn/bus/line!busesDetail.action?' + 'sign=3BszHrCL%2BYb8Ix6to1kyJw%3D%3D&lineName=773&cityId=040' + '&lineNo=07730&stationName=%E5%87%8C%E5%A1%98%E6%9D%911&cshow=linedetail' + '&targetStationLat=23.173592746309094&targetOrder=6&targetStationLng=113.40505964506974' + '&lineId=020-07730-0&s=android&last_src=app_qq_sj&v=3.79.2') + resp2 = requests.post('https://api.chelaile.net.cn/bus/line!busesDetail.action?sign=CMkEe%2BVN716ghdReiM5G9Q%3D%3D&cshow=linedetail&nw=WIFI&stats_referer=searchResult&lineName=773&language=3&cityId=040&AndroidID=3a991014707e4ab7&mac=14%3Aa5%3A1a%3Ac0%3A96%3Ae1&lineNo=07730&phoneBrand=HUAWEI&stationName=%E5%AE%B6%E5%AE%B6%E4%B9%90%E5%8C%BB%E9%99%A22&lchsrc=icon&udid=d7317fff-cc3c-4daa-b767-db9cc1d3b8e0&targetStationLat=23.135617742613036&system_push_open=1&direction=1&deviceType=WAS-AL00&targetOrder=4&push_open=1&o1=a885683212c1a4af371435766be40e941a0b9c26&sv=8.0.0&targetStationLng=113.40046849892799&first_src=app_qq_sj&screenHeight=1848&lineId=020-07730-1&userAgent=Mozilla%2F5.0+%28Linux%3B+Android+8.0.0%3B+WAS-AL00+Build%2FHUAWEIWAS-AL00%3B+wv%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Version%2F4.0+Chrome%2F70.0.3538.110+Mobile+Safari%2F537.36&vc=146&userId=unknown&isNewLineDetail=1&filter=1&s=android&last_src=app_qq_sj&v=3.79.2&wifi_open=1&imei=863095033108438&stats_order=1-1&screenDensity=3.0&flpolicy=0') + time.sleep(30) + + +itchat.auto_login(hotReload=True, enableCmdQR=2) +itchat.run() From 789b3b4567ad421a76f5bb5a41ba2d283f4832fd Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 24 Jun 2019 14:16:28 +0800 Subject: [PATCH 073/148] update tip --- wechat/chelaile.py | 28 ------------------------- wechat/tip.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 28 deletions(-) delete mode 100644 wechat/chelaile.py create mode 100644 wechat/tip.py diff --git a/wechat/chelaile.py b/wechat/chelaile.py deleted file mode 100644 index 3994957..0000000 --- a/wechat/chelaile.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# @author : jared -# @date : 2019/6/22 17:39 - -import time -import json -import itchat -import requests - - -@itchat.msg_register(itchat.content.TEXT) -def text(msg): - receive_text = msg['Text'] - to_user_id = msg['ToUserName'] # 接收人 - from_user_id = msg['FromUserName'] # 发送人 - - while receive_text == "1": - resp = requests.post('https://api.chelaile.net.cn/bus/line!busesDetail.action?' - 'sign=3BszHrCL%2BYb8Ix6to1kyJw%3D%3D&lineName=773&cityId=040' - '&lineNo=07730&stationName=%E5%87%8C%E5%A1%98%E6%9D%911&cshow=linedetail' - '&targetStationLat=23.173592746309094&targetOrder=6&targetStationLng=113.40505964506974' - '&lineId=020-07730-0&s=android&last_src=app_qq_sj&v=3.79.2') - resp2 = requests.post('https://api.chelaile.net.cn/bus/line!busesDetail.action?sign=CMkEe%2BVN716ghdReiM5G9Q%3D%3D&cshow=linedetail&nw=WIFI&stats_referer=searchResult&lineName=773&language=3&cityId=040&AndroidID=3a991014707e4ab7&mac=14%3Aa5%3A1a%3Ac0%3A96%3Ae1&lineNo=07730&phoneBrand=HUAWEI&stationName=%E5%AE%B6%E5%AE%B6%E4%B9%90%E5%8C%BB%E9%99%A22&lchsrc=icon&udid=d7317fff-cc3c-4daa-b767-db9cc1d3b8e0&targetStationLat=23.135617742613036&system_push_open=1&direction=1&deviceType=WAS-AL00&targetOrder=4&push_open=1&o1=a885683212c1a4af371435766be40e941a0b9c26&sv=8.0.0&targetStationLng=113.40046849892799&first_src=app_qq_sj&screenHeight=1848&lineId=020-07730-1&userAgent=Mozilla%2F5.0+%28Linux%3B+Android+8.0.0%3B+WAS-AL00+Build%2FHUAWEIWAS-AL00%3B+wv%29+AppleWebKit%2F537.36+%28KHTML%2C+like+Gecko%29+Version%2F4.0+Chrome%2F70.0.3538.110+Mobile+Safari%2F537.36&vc=146&userId=unknown&isNewLineDetail=1&filter=1&s=android&last_src=app_qq_sj&v=3.79.2&wifi_open=1&imei=863095033108438&stats_order=1-1&screenDensity=3.0&flpolicy=0') - time.sleep(30) - - -itchat.auto_login(hotReload=True, enableCmdQR=2) -itchat.run() diff --git a/wechat/tip.py b/wechat/tip.py new file mode 100644 index 0000000..74eb214 --- /dev/null +++ b/wechat/tip.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# @author : Jared +# @date : 2019/6/22 17:39 + +import time +import json +import itchat +import requests +from datetime import datetime + + +class CheLaiLe: + start = False + + +@itchat.msg_register(itchat.content.TEXT) +def text(msg): + receive_text = msg['Text'] + to_user_id = msg['ToUserName'] # 接收人 + # from_user_id = msg['FromUserName'] # 发送人 + + if to_user_id == 'filehelper': + if receive_text.startswith("b"): + # 车来了 + CheLaiLe.start = not CheLaiLe.start + while CheLaiLe.start: + if datetime.now().hour < 12: + resp = requests.get( + 'https://api.chelaile.net.cn/bus/line!busesDetail.action' + '?sign=3BszHrCL%2BYb8Ix6to1kyJw%3D%3D&v=3.79.2&s=android' + '&cityId=040&targetOrder=6&lineId=020-07730-0').text + else: + resp = requests.get( + 'https://api.chelaile.net.cn/bus/line!busesDetail.action' + '?sign=CMkEe%2BVN716ghdReiM5G9Q%3D%3D&s=android&v=3.79.2' + '&cityId=040&targetOrder=4&lineId=020-07730-1').text + resp = resp.replace("**YGKJ", "").replace("YGKJ##", "") + json_content = json.loads(resp) + data = json_content.get('jsonr').get("data") + tip_desc, pre_arrival_desc = data.get("tip").get("desc"), data.get("depDesc") + + send_msg = "{}\n[{}]".format(tip_desc, pre_arrival_desc) + itchat.send(send_msg, toUserName='filehelper') + time.sleep(30) + + if receive_text.startswith("s"): + # 贝壳小岛 + pass + + +itchat.auto_login(hotReload=True) +itchat.run() From 3140627cfeeb0b1361df6df8139350e6c979d20c Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 24 Jun 2019 14:24:36 +0800 Subject: [PATCH 074/148] update tip --- wechat/tip.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/wechat/tip.py b/wechat/tip.py index 74eb214..6d3db42 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -13,6 +13,10 @@ class CheLaiLe: start = False +class ShellIsland: + start = False + + @itchat.msg_register(itchat.content.TEXT) def text(msg): receive_text = msg['Text'] @@ -45,7 +49,10 @@ def text(msg): if receive_text.startswith("s"): # 贝壳小岛 - pass + ShellIsland.start = not ShellIsland.start + while ShellIsland.start: + itchat.send("", toUserName='filehelper') + time.sleep(60) itchat.auto_login(hotReload=True) From ab802a2371b96b3d0a12d2308363db4a36260308 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 24 Jun 2019 14:26:37 +0800 Subject: [PATCH 075/148] update tip --- wechat/tip.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wechat/tip.py b/wechat/tip.py index 6d3db42..83ec0e9 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -17,13 +17,16 @@ class ShellIsland: start = False +file_helper = 'filehelper' + + @itchat.msg_register(itchat.content.TEXT) def text(msg): receive_text = msg['Text'] to_user_id = msg['ToUserName'] # 接收人 # from_user_id = msg['FromUserName'] # 发送人 - if to_user_id == 'filehelper': + if to_user_id == file_helper: if receive_text.startswith("b"): # 车来了 CheLaiLe.start = not CheLaiLe.start @@ -44,14 +47,14 @@ def text(msg): tip_desc, pre_arrival_desc = data.get("tip").get("desc"), data.get("depDesc") send_msg = "{}\n[{}]".format(tip_desc, pre_arrival_desc) - itchat.send(send_msg, toUserName='filehelper') + itchat.send(send_msg, toUserName=file_helper) time.sleep(30) if receive_text.startswith("s"): # 贝壳小岛 ShellIsland.start = not ShellIsland.start while ShellIsland.start: - itchat.send("", toUserName='filehelper') + itchat.send("", toUserName=file_helper) time.sleep(60) From 2970ef82da07e09b0008bcd275f6f893365324c6 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 24 Jun 2019 14:34:32 +0800 Subject: [PATCH 076/148] update tip --- wechat/tip.py | 2 +- wechat/tip.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 wechat/tip.sh diff --git a/wechat/tip.py b/wechat/tip.py index 83ec0e9..623a7ba 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -58,5 +58,5 @@ def text(msg): time.sleep(60) -itchat.auto_login(hotReload=True) +itchat.auto_login(hotReload=True, enableCmdQR=2) itchat.run() diff --git a/wechat/tip.sh b/wechat/tip.sh new file mode 100644 index 0000000..b42047a --- /dev/null +++ b/wechat/tip.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +nohup python3 tip.py >> /dev/null 2>&1 & \ No newline at end of file From 23b328725f024f380ce8926523a06aaf5d0a4916 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 24 Jun 2019 23:21:51 +0800 Subject: [PATCH 077/148] init chelaile --- spider/jianshu/robot.py | 3 ++- wechat/tip.py | 13 +++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index aeab0de..63b77d3 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -114,7 +114,8 @@ def sleep(min_seconds=3, max_seconds=10): # 4.评论 jianshu_p = "{}/p/".format(jianshu) comments = {"ac02c56c0865": "3200,38968576", "be27870bdba9": "1400,38969657", "277f1b0a140d": "1400,40692862", - "15eb212788fc": "2150,40693328", "81d5ae85f38a": "3550,40874580", "4dca23c2c75a": "3550,40874520"} + "15eb212788fc": "2150,40693328", "81d5ae85f38a": "3550,40874580", "4dca23c2c75a": "3550,40874520", + "9aaebad1753a": "3500,42635258"} for comment_id, info in comments.items(): driver.get("{}{}".format(jianshu_p, comment_id)), sleep() y_scroll, button_id = info.split(",") diff --git a/wechat/tip.py b/wechat/tip.py index 623a7ba..09afa07 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -2,15 +2,16 @@ # @author : Jared # @date : 2019/6/22 17:39 -import time import json +import time +from datetime import datetime + import itchat import requests -from datetime import datetime class CheLaiLe: - start = False + count, step, max_count = 0, 1, 90 class ShellIsland: @@ -29,8 +30,8 @@ def text(msg): if to_user_id == file_helper: if receive_text.startswith("b"): # 车来了 - CheLaiLe.start = not CheLaiLe.start - while CheLaiLe.start: + while CheLaiLe.count <= CheLaiLe.max_count: + CheLaiLe.count += CheLaiLe.step if datetime.now().hour < 12: resp = requests.get( 'https://api.chelaile.net.cn/bus/line!busesDetail.action' @@ -58,5 +59,5 @@ def text(msg): time.sleep(60) -itchat.auto_login(hotReload=True, enableCmdQR=2) +itchat.auto_login(hotReload=True) itchat.run() From e2e5a943698ac90b83aaae762db887f4f9276881 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 25 Jun 2019 10:05:49 +0800 Subject: [PATCH 078/148] fix tip bug --- wechat/tip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wechat/tip.py b/wechat/tip.py index 09afa07..6983d94 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -30,6 +30,7 @@ def text(msg): if to_user_id == file_helper: if receive_text.startswith("b"): # 车来了 + CheLaiLe.count = 0 while CheLaiLe.count <= CheLaiLe.max_count: CheLaiLe.count += CheLaiLe.step if datetime.now().hour < 12: From 613eb361f06869d4a46e61e151f7e2e6bd168823 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 26 Jun 2019 17:40:28 +0800 Subject: [PATCH 079/148] update jianshu start --- spider/jianshu/start.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py index 5b999a7..d1e4202 100644 --- a/spider/jianshu/start.py +++ b/spider/jianshu/start.py @@ -21,4 +21,7 @@ def robot(): # nohup python3 start.py 22 >> /dev/null 2>&1 & if __name__ == '__main__': argv = sys.argv - robot_job(argv[1]) + if len(argv) > 1: + robot_job(argv[1]) + else: + robot_job(6) From 0f335cf4968924a95b5202b3ac3cb53dd0265af7 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 28 Jun 2019 14:36:43 +0800 Subject: [PATCH 080/148] add tip job --- wechat/tip.sh | 2 -- wechat/tipjob.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) delete mode 100644 wechat/tip.sh create mode 100644 wechat/tipjob.py diff --git a/wechat/tip.sh b/wechat/tip.sh deleted file mode 100644 index b42047a..0000000 --- a/wechat/tip.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -nohup python3 tip.py >> /dev/null 2>&1 & \ No newline at end of file diff --git a/wechat/tipjob.py b/wechat/tipjob.py new file mode 100644 index 0000000..d722b6c --- /dev/null +++ b/wechat/tipjob.py @@ -0,0 +1,18 @@ +import os +from apscheduler.schedulers.background import * + + +def job_task(): + scheduler = BlockingScheduler() + scheduler.add_job(tip, 'cron', hour=19, minute="30,45,59") + scheduler.start() + + +def tip(): + resp = os.popen("ps -ef|grep python | grep tip | wc -l").read() + if int(resp) > 1: + os.system("nohup python3 tip.py >> /dev/null 2>&1 &") + + +if __name__ == '__main__': + job_task() From 69a7067c6d585a8d16236a96bf6c74d1314f8cee Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 28 Jun 2019 14:42:44 +0800 Subject: [PATCH 081/148] add tip job --- wechat/tipjob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wechat/tipjob.py b/wechat/tipjob.py index d722b6c..53aac2b 100644 --- a/wechat/tipjob.py +++ b/wechat/tipjob.py @@ -9,7 +9,7 @@ def job_task(): def tip(): - resp = os.popen("ps -ef|grep python | grep tip | wc -l").read() + resp = os.popen("ps -ef|grep python | grep tip.py | wc -l").read() if int(resp) > 1: os.system("nohup python3 tip.py >> /dev/null 2>&1 &") From c71684e4272479ac2eec87571a8007fa74343482 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 28 Jun 2019 14:44:51 +0800 Subject: [PATCH 082/148] add tip job --- wechat/tipjob.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/wechat/tipjob.py b/wechat/tipjob.py index 53aac2b..15b9cbe 100644 --- a/wechat/tipjob.py +++ b/wechat/tipjob.py @@ -10,8 +10,11 @@ def job_task(): def tip(): resp = os.popen("ps -ef|grep python | grep tip.py | wc -l").read() - if int(resp) > 1: + if int(resp) <= 1: + print("脚本即将运行...") os.system("nohup python3 tip.py >> /dev/null 2>&1 &") + else: + print("脚本已经运行...") if __name__ == '__main__': From 151266077bad61bacdf22accff4a50a39984dabf Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 29 Jun 2019 11:02:17 +0800 Subject: [PATCH 083/148] update comments id --- spider/jianshu/robot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 63b77d3..ccfb50c 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -115,7 +115,9 @@ def sleep(min_seconds=3, max_seconds=10): jianshu_p = "{}/p/".format(jianshu) comments = {"ac02c56c0865": "3200,38968576", "be27870bdba9": "1400,38969657", "277f1b0a140d": "1400,40692862", "15eb212788fc": "2150,40693328", "81d5ae85f38a": "3550,40874580", "4dca23c2c75a": "3550,40874520", - "9aaebad1753a": "3500,42635258"} + "9aaebad1753a": "3500,42635258", + "f3398c00ffc1": "1550,42854890", "e369bb81bcc8": "1500,42855033", "4ec116446717": "1300,42855114", + "2d184d128522": "1400,42855181", } for comment_id, info in comments.items(): driver.get("{}{}".format(jianshu_p, comment_id)), sleep() y_scroll, button_id = info.split(",") From 3589d159d0dd07903c48e0ca2217695b1a7477dd Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 30 Jun 2019 09:26:11 +0800 Subject: [PATCH 084/148] update jianshu robot --- spider/jianshu/robot.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index ccfb50c..c2aa191 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -113,15 +113,12 @@ def sleep(min_seconds=3, max_seconds=10): # 4.评论 jianshu_p = "{}/p/".format(jianshu) - comments = {"ac02c56c0865": "3200,38968576", "be27870bdba9": "1400,38969657", "277f1b0a140d": "1400,40692862", - "15eb212788fc": "2150,40693328", "81d5ae85f38a": "3550,40874580", "4dca23c2c75a": "3550,40874520", - "9aaebad1753a": "3500,42635258", - "f3398c00ffc1": "1550,42854890", "e369bb81bcc8": "1500,42855033", "4ec116446717": "1300,42855114", - "2d184d128522": "1400,42855181", } - for comment_id, info in comments.items(): - driver.get("{}{}".format(jianshu_p, comment_id)), sleep() - y_scroll, button_id = info.split(",") - driver.execute_script("window.scrollTo(0,{})".format(y_scroll)), sleep(3, 5) + comments = {"ac02c56c0865": "38968576", "be27870bdba9": "38969657", "277f1b0a140d": "40692862", + "15eb212788fc": "40693328", "81d5ae85f38a": "40874580", "4dca23c2c75a": "40874520", + "9aaebad1753a": "42635258", "f3398c00ffc1": "42854890", "e369bb81bcc8": "42855033", + "4ec116446717": "42855114", "2d184d128522": "42855181", } + for comment_id, button_id in comments.items(): + driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)), sleep(3, 5) button_element = 'like-button-{}'.format(button_id) try: driver.find_element_by_id(button_element).click(), sleep(3, 5) From 3d0e4cfd920a305419ccace628aa1dcf42d97dd2 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 1 Jul 2019 20:39:54 +0800 Subject: [PATCH 085/148] update tip job --- wechat/tip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wechat/tip.py b/wechat/tip.py index 6983d94..a66041a 100644 --- a/wechat/tip.py +++ b/wechat/tip.py @@ -60,5 +60,5 @@ def text(msg): time.sleep(60) -itchat.auto_login(hotReload=True) +itchat.auto_login(hotReload=True, enableCmdQR=2) itchat.run() From 26809182eb0def9f18a256144b4a1714725efd2d Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 5 Jul 2019 09:46:53 +0800 Subject: [PATCH 086/148] update jianshu robot --- spider/jianshu/robot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index c2aa191..070c7bd 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -16,7 +16,7 @@ role = int(argv[3]) if argv_length > 3 else input("请输入Role\n") if platform.system() == "Linux": - os.system('ps -ef | grep chrome | awk "{print \$2}" | xargs kill -9') + os.system("ps -ef | grep chrome | awk '{print $2}' | xargs kill -9") def sleep(min_seconds=3, max_seconds=10): @@ -57,17 +57,18 @@ def sleep(min_seconds=3, max_seconds=10): try: driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() except: - logging.info("{} : 登录失败".format(role)) + logging.error("{} : 登录失败".format(role)) else: sleep() page_title = driver.title - logging.info("{} ：{}".format(role, page_title)) + if page_title.find("简书") == -1: + logging.error("{} ：{}".format(role, page_title)) # 1. 收益 driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) elements = driver.find_elements_by_class_name("order") - info = ' ; '.join(map(lambda ele: ele.text, elements[1:])) + info = ' ; '.join(map(lambda ele: ele.text, elements)) logging.info("{} : {}".format(role, info)) sleep() @@ -124,6 +125,7 @@ def sleep(min_seconds=3, max_seconds=10): driver.find_element_by_id(button_element).click(), sleep(3, 5) driver.find_element_by_id(button_element).click() except: + logging.error("Button_Element : {}".format(button_element)) pass finally: driver.close() From 472284b044b9b04c403df38af224de166071dd37 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 16 Jul 2019 08:30:29 +0800 Subject: [PATCH 087/148] init like --- spider/jianshu/like.py | 108 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 spider/jianshu/like.py diff --git a/spider/jianshu/like.py b/spider/jianshu/like.py new file mode 100644 index 0000000..db6bbbc --- /dev/null +++ b/spider/jianshu/like.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# @author : jared +# @date : 2019/7/5 23:23 + +import logging +import random +import time +from bs4 import BeautifulSoup + +import requests +from selenium import webdriver +from selenium.webdriver.chrome.options import Options + +logging.basicConfig(level='INFO', filename='info.log', format='%(message)s') +headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} +jianshu = "https://www.jianshu.com" + +articles = set() +user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", "b34dcc8c2394", "37ec6dd9fc1c", "04f7029dba14"] +jianshu_u = "{}/u/".format(jianshu) +for uid in user_ids: + user = "{}{}".format(jianshu_u, uid) + html = requests.get(user, headers=headers).text + bs = BeautifulSoup(html, 'html.parser') + info = bs.find("div", class_="info") + p_list = info.find_all("p") + article_num = int(p_list[2].text) + + count = 0 + while count < article_num: + for page in range(1, 100): + html = requests.get("{}?order_by=shared_at&page={}".format(user, page), headers=headers).text + bs = BeautifulSoup(html, 'html.parser') + titles = bs.find_all('a', class_="title") + for title in titles: + article_url = title.get("href") + if article_url: + count += 1 + articles.add(article_url) + if count >= article_num: + break + + +def sleep(min_seconds=3, max_seconds=10): + """随机休眠""" + time.sleep(random.randint(min_seconds, max_seconds)) + + +data = {"269373": "6"} +for username, role in data.items(): + + # 无头模式 + password = "" + options = Options() + if role != "6": + options.add_argument('--headless') + options.add_argument('log-level=3') + options.add_argument('--no-sandbox') + options.add_argument('--disable-gpu') + options.add_argument('window-size=1366x728') + options.add_argument('--disable-dev-shm-usage') + options.add_argument('blink-settings=imagesEnabled=false') + driver = webdriver.Chrome(chrome_options=options) + + # 登录简书 + driver.get("{}/sign_in".format(jianshu)) + driver.maximize_window() + + if role != "6": + # 豆瓣登录 + driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() + driver.find_element_by_class_name("douban").click() + sleep() + + # 切换窗口 + driver.close() + window_handles = driver.window_handles + driver.switch_to.window(window_handles[-1]) + driver.find_element_by_id("inp-alias").send_keys(username) + driver.find_element_by_id("inp-pwd").send_keys(password) + + try: + if role != "6": + driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() + except: + logging.error("{} : 登录失败".format(role)) + else: + if role != "6": + sleep() + else: + sleep(45, 60) + + page_title = driver.title + if page_title.find("简书") == -1: + logging.error("{} ：{}".format(role, page_title)) + else: + logging.info("Role: {}".format(role)) + for article in articles: + url = "{}{}".format(jianshu, article) + driver.get(url) + bs = BeautifulSoup(driver.page_source, 'html.parser') + like_element = bs.find("div", class_="btn like-group active") + if not like_element: + logging.info("{}".format(url)) + + finally: + driver.close() + driver.quit() From 912b5a619b2471587c31dce14cc48f4a63d6fcce Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 17 Jul 2019 11:34:27 +0800 Subject: [PATCH 088/148] init jianshuium --- spider/jianshu/jianshuium.py | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 spider/jianshu/jianshuium.py diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/jianshuium.py new file mode 100644 index 0000000..8ee9059 --- /dev/null +++ b/spider/jianshu/jianshuium.py @@ -0,0 +1,52 @@ +from appium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC + +server = 'http://localhost:4723/wd/hub' # Appium Server, 端口默认为4723 +desired_capabilities = { + 'platformName': 'Android', + 'deviceName': 'WAS_AL00', # 需替换成你的deviceName + 'appPackage': 'com.gotokeep.keep', + 'appActivity': 'com.gotokeep.keep.splash.SplashActivity' +} + +driver = webdriver.Remote(server, desired_capabilities) +wait = WebDriverWait(driver, 10) # 最大查找等待超时时间：10s + + +def get_permission(): + """允许APP获取的某些权限""" + + try: + ask = wait.until(EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/do_not_ask_checkbox'))) + ask.click() + allow = wait.until( + EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/permission_allow_button'))) + allow.click() + except: + pass + + +# 允许两项授权 +get_permission() +get_permission() + +# 点击“立即使用” +welcome = wait.until(EC.presence_of_element_located((By.ID, 'com.gotokeep.keep:id/btn_bottom_in_video_welcome'))) +welcome.click() + +# 切换“密码登录”(同样可以使用第三方进行授权登录) +driver.tap([(900, 110)]) + +# 输入“手机号” +phone = driver.find_element_by_accessibility_id('Phone Number In Login') +phone.send_keys('13988888888') # 替换成实际的账号 + +# 输入“密码” +password = driver.find_element_by_accessibility_id('Password In Login') +password.send_keys('123456') # 替换成实际的密码 + +# 点击“登录” +login = driver.find_element_by_id('com.gotokeep.keep:id/btn_action') +login.click() From e0bb38b7335faf7bd52e4853c7d91033deb01ec3 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 17 Jul 2019 22:30:32 +0800 Subject: [PATCH 089/148] init jianshuium --- spider/jianshu/jianshuium.py | 78 +++++++++++++++++------------------- 1 file changed, 36 insertions(+), 42 deletions(-) diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/jianshuium.py index 8ee9059..89505ed 100644 --- a/spider/jianshu/jianshuium.py +++ b/spider/jianshu/jianshuium.py @@ -1,52 +1,46 @@ +import time from appium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC -server = 'http://localhost:4723/wd/hub' # Appium Server, 端口默认为4723 +# Appium Server, 端口默认为4723 + +server = 'http://localhost:4723/wd/hub' desired_capabilities = { - 'platformName': 'Android', - 'deviceName': 'WAS_AL00', # 需替换成你的deviceName - 'appPackage': 'com.gotokeep.keep', - 'appActivity': 'com.gotokeep.keep.splash.SplashActivity' + "platformName": "Android", + "deviceName": "WAS_AL00", + "appPackage": "com.jianshu.haruki", + "appActivity": "com.baiji.jianshu.MainActivity" } driver = webdriver.Remote(server, desired_capabilities) -wait = WebDriverWait(driver, 10) # 最大查找等待超时时间：10s - - -def get_permission(): - """允许APP获取的某些权限""" - - try: - ask = wait.until(EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/do_not_ask_checkbox'))) - ask.click() - allow = wait.until( - EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/permission_allow_button'))) - allow.click() - except: - pass - - -# 允许两项授权 -get_permission() -get_permission() - -# 点击“立即使用” -welcome = wait.until(EC.presence_of_element_located((By.ID, 'com.gotokeep.keep:id/btn_bottom_in_video_welcome'))) -welcome.click() - -# 切换“密码登录”(同样可以使用第三方进行授权登录) -driver.tap([(900, 110)]) - -# 输入“手机号” -phone = driver.find_element_by_accessibility_id('Phone Number In Login') -phone.send_keys('13988888888') # 替换成实际的账号 - -# 输入“密码” -password = driver.find_element_by_accessibility_id('Password In Login') -password.send_keys('123456') # 替换成实际的密码 - -# 点击“登录” -login = driver.find_element_by_id('com.gotokeep.keep:id/btn_action') +wait = WebDriverWait(driver, 10) + +# 点击“我的” +ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_mine'))) +ele.click() +ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) +ele.click() + +# 登录 +ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) +ele.click() +ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) +ele.send_keys('') +ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) +ele.send_keys('') +login = driver.find_element_by_id('com.jianshu.haruki:id/tv_login') login.click() + +time.sleep(3) + +driver.tap([(535, 1810)]) +time.sleep(10) +driver.tap([(525, 1530)]) +time.sleep(5) +while True: + driver.tap([(525, 1160)]) + time.sleep(35) + driver.tap([(980, 150)]) + driver.tap([(545, 1020)]) From 218552c2c995c671c07cc3e5741c99a395e7e1af Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 00:03:27 +0800 Subject: [PATCH 090/148] init jianshuium --- spider/jianshu/jianshuium.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/jianshuium.py index 89505ed..7e471f9 100644 --- a/spider/jianshu/jianshuium.py +++ b/spider/jianshu/jianshuium.py @@ -41,6 +41,8 @@ time.sleep(5) while True: driver.tap([(525, 1160)]) - time.sleep(35) - driver.tap([(980, 150)]) + time.sleep(3) driver.tap([(545, 1020)]) + time.sleep(35) + driver.tap([(990, 100)]) + time.sleep(3) From eb503f7ebfd8581b12f9aedd2877642c6c9db7c1 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 13:42:43 +0800 Subject: [PATCH 091/148] update jianshuium --- spider/jianshu/jianshuium.py | 134 ++++++++++++++++++++++++++--------- 1 file changed, 99 insertions(+), 35 deletions(-) diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/jianshuium.py index 7e471f9..ee3f6fb 100644 --- a/spider/jianshu/jianshuium.py +++ b/spider/jianshu/jianshuium.py @@ -1,10 +1,9 @@ import time + from appium import webdriver from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC - -# Appium Server, 端口默认为4723 +from selenium.webdriver.support.ui import WebDriverWait server = 'http://localhost:4723/wd/hub' desired_capabilities = { @@ -14,35 +13,100 @@ "appActivity": "com.baiji.jianshu.MainActivity" } -driver = webdriver.Remote(server, desired_capabilities) -wait = WebDriverWait(driver, 10) - -# 点击“我的” -ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_mine'))) -ele.click() -ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) -ele.click() - -# 登录 -ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) -ele.click() -ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) -ele.send_keys('') -ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) -ele.send_keys('') -login = driver.find_element_by_id('com.jianshu.haruki:id/tv_login') -login.click() - -time.sleep(3) - -driver.tap([(535, 1810)]) -time.sleep(10) -driver.tap([(525, 1530)]) -time.sleep(5) -while True: - driver.tap([(525, 1160)]) - time.sleep(3) - driver.tap([(545, 1020)]) - time.sleep(35) - driver.tap([(990, 100)]) - time.sleep(3) +data = {"269373": "6"} +for username, password in data.items(): + driver = webdriver.Remote(server, desired_capabilities) + wait = WebDriverWait(driver, 30) + short_wait = WebDriverWait(driver, 10) + long_wait = WebDriverWait(driver, 60) + + # 我的 + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) + ele.click() + + # 登录 + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) + ele.send_keys(username) + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) + ele.send_keys(password) + login = driver.find_element_by_id('com.jianshu.haruki:id/tv_login') + login.click() + + # 简书钻 -> 抽奖 + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) + ele.click() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.webkit.WebView/android.webkit.WebView/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[17]/android.view.View[3]'))) + ele.click() + + # 每日抽奖 + while True: + # 1. GO + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout' + '/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) + ele.click() + try: + # 2. 继续观看 + ele = short_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.view.View[2]'))) + + except: + pass + else: + ele.click() + # 根据元素是否还可以点击来判断是否还有抽奖次数 todo + # 3. 关闭 + ele = long_wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + try: + # 4. 知道了 todo + time.sleep(5) + driver.tap([(540, 1335)]) + except: + pass From 5aa8c64bbdf4b54c78a527c53aefcdcbfa2b977b Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 14:01:36 +0800 Subject: [PATCH 092/148] update jianshu --- spider/jianshu/data_.py | 13 +++++++++++++ spider/jianshu/jianshuium.py | 7 +++++-- spider/jianshu/like.py | 10 ++++++---- 3 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 spider/jianshu/data_.py diff --git a/spider/jianshu/data_.py b/spider/jianshu/data_.py new file mode 100644 index 0000000..fee9f26 --- /dev/null +++ b/spider/jianshu/data_.py @@ -0,0 +1,13 @@ +import csv + + +def load_data(): + """ 加载数据 """ + + print("开始读取") + data = csv.reader(open('D:\\data.csv', encoding='utf-8')) + for row in data: + username, password, role = row + print("Username: {} Password: {} Role: {}".format(username, password, role)) + print("读取结束") + return data diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/jianshuium.py index ee3f6fb..dd558c6 100644 --- a/spider/jianshu/jianshuium.py +++ b/spider/jianshu/jianshuium.py @@ -1,4 +1,5 @@ import time +from . import data_ from appium import webdriver from selenium.webdriver.common.by import By @@ -13,8 +14,10 @@ "appActivity": "com.baiji.jianshu.MainActivity" } -data = {"269373": "6"} -for username, password in data.items(): +data = data_.load_data() +for row in data: + username, password, role = row + driver = webdriver.Remote(server, desired_capabilities) wait = WebDriverWait(driver, 30) short_wait = WebDriverWait(driver, 10) diff --git a/spider/jianshu/like.py b/spider/jianshu/like.py index db6bbbc..ee760ff 100644 --- a/spider/jianshu/like.py +++ b/spider/jianshu/like.py @@ -5,12 +5,14 @@ import logging import random import time -from bs4 import BeautifulSoup import requests +from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options +from . import data_ + logging.basicConfig(level='INFO', filename='info.log', format='%(message)s') headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} jianshu = "https://www.jianshu.com" @@ -46,11 +48,11 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) -data = {"269373": "6"} -for username, role in data.items(): +data = data_.load_data() +for row in data: + username, password, role = row # 无头模式 - password = "" options = Options() if role != "6": options.add_argument('--headless') From 08336fd534c644fb29678e19d867694c85b1b799 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 14:34:58 +0800 Subject: [PATCH 093/148] update jianshu --- spider/jianshu/data_.py | 24 +++++++++++++++++++++++- spider/jianshu/like.py | 27 ++++++++++++++++++--------- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/spider/jianshu/data_.py b/spider/jianshu/data_.py index fee9f26..3ac82a3 100644 --- a/spider/jianshu/data_.py +++ b/spider/jianshu/data_.py @@ -1,13 +1,35 @@ import csv +import os + +base = "D:\\Data\\" def load_data(): """ 加载数据 """ print("开始读取") - data = csv.reader(open('D:\\data.csv', encoding='utf-8')) + data = csv.reader(open('{}data.csv'.format(base), encoding='utf-8')) for row in data: username, password, role = row print("Username: {} Password: {} Role: {}".format(username, password, role)) print("读取结束") return data + + +def load_article(role): + """ 加载文章 """ + + print("开始读取") + path = "{}{}".format(base, role) + file_list = os.listdir(path) + print("文件夹路径【{}】下共有文章{}篇".format(path, len(file_list))) + data = dict() + for file_name in file_list: + article_name = file_name.replace(".md", "") + with open(path + "\\" + file_name, "r", encoding='utf-8') as file: + article_content = file.read() + data.setdefault(article_name, article_content) + print("读取结束".format(role)) + return data + + diff --git a/spider/jianshu/like.py b/spider/jianshu/like.py index ee760ff..21061cf 100644 --- a/spider/jianshu/like.py +++ b/spider/jianshu/like.py @@ -5,15 +5,17 @@ import logging import random import time +import pyperclip import requests from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.keys import Keys from . import data_ -logging.basicConfig(level='INFO', filename='info.log', format='%(message)s') +logging.basicConfig(level='INFO', filename='like.log', format='%(message)s') headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} jianshu = "https://www.jianshu.com" @@ -44,15 +46,13 @@ def sleep(min_seconds=3, max_seconds=10): - """随机休眠""" time.sleep(random.randint(min_seconds, max_seconds)) data = data_.load_data() for row in data: - username, password, role = row - # 无头模式 + username, password, role = row options = Options() if role != "6": options.add_argument('--headless') @@ -64,17 +64,14 @@ def sleep(min_seconds=3, max_seconds=10): options.add_argument('blink-settings=imagesEnabled=false') driver = webdriver.Chrome(chrome_options=options) - # 登录简书 driver.get("{}/sign_in".format(jianshu)) driver.maximize_window() if role != "6": - # 豆瓣登录 driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() driver.find_element_by_class_name("douban").click() sleep() - # 切换窗口 driver.close() window_handles = driver.window_handles driver.switch_to.window(window_handles[-1]) @@ -85,7 +82,7 @@ def sleep(min_seconds=3, max_seconds=10): if role != "6": driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() except: - logging.error("{} : 登录失败".format(role)) + logging.error("{} : 授权登录失败".format(role)) else: if role != "6": sleep() @@ -94,9 +91,11 @@ def sleep(min_seconds=3, max_seconds=10): page_title = driver.title if page_title.find("简书") == -1: - logging.error("{} ：{}".format(role, page_title)) + logging.error("{} ：{} 登录失败".format(role, page_title)) else: logging.info("Role: {}".format(role)) + + # 1. 喜欢 for article in articles: url = "{}{}".format(jianshu, article) driver.get(url) @@ -105,6 +104,16 @@ def sleep(min_seconds=3, max_seconds=10): if not like_element: logging.info("{}".format(url)) + # 2. 写文 + article_data = data_.load_article(role) + driver.get("{}/writer#/".format(jianshu)), sleep() + for title, content in article_data.items(): + driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() + driver.find_element_by_class_name("_24i7u").clear() + driver.find_element_by_class_name("_24i7u").send_keys(title), sleep() + driver.find_element_by_id("arthur-editor").clear() + pyperclip.copy(content) + driver.find_element_by_id("arthur-editor").send_keys(Keys.CONTROL, 'v'), sleep() finally: driver.close() driver.quit() From b2cd1e4d84d93eae3c036f80a3f40cfaf0821713 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 14:36:42 +0800 Subject: [PATCH 094/148] refactor jianshu --- spider/jianshu/like.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/like.py b/spider/jianshu/like.py index 21061cf..295f7c1 100644 --- a/spider/jianshu/like.py +++ b/spider/jianshu/like.py @@ -5,8 +5,8 @@ import logging import random import time -import pyperclip +import pyperclip import requests from bs4 import BeautifulSoup from selenium import webdriver From bdf0a4c285710a066b983bad9da0457655e3b265 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 14:39:41 +0800 Subject: [PATCH 095/148] refactor jianshu --- spider/jianshu/{jianshuium.py => aium.py} | 4 +- spider/jianshu/{data_.py => load.py} | 6 +- spider/jianshu/robot.py | 96 ++++++------------ spider/jianshu/{like.py => sium.py} | 6 +- spider/jianshu/spider.py | 118 ---------------------- 5 files changed, 39 insertions(+), 191 deletions(-) rename spider/jianshu/{jianshuium.py => aium.py} (99%) rename spider/jianshu/{data_.py => load.py} (95%) rename spider/jianshu/{like.py => sium.py} (97%) delete mode 100644 spider/jianshu/spider.py diff --git a/spider/jianshu/jianshuium.py b/spider/jianshu/aium.py similarity index 99% rename from spider/jianshu/jianshuium.py rename to spider/jianshu/aium.py index dd558c6..4374a9e 100644 --- a/spider/jianshu/jianshuium.py +++ b/spider/jianshu/aium.py @@ -1,5 +1,5 @@ import time -from . import data_ +from . import load from appium import webdriver from selenium.webdriver.common.by import By @@ -14,7 +14,7 @@ "appActivity": "com.baiji.jianshu.MainActivity" } -data = data_.load_data() +data = load.load_account() for row in data: username, password, role = row diff --git a/spider/jianshu/data_.py b/spider/jianshu/load.py similarity index 95% rename from spider/jianshu/data_.py rename to spider/jianshu/load.py index 3ac82a3..58609c1 100644 --- a/spider/jianshu/data_.py +++ b/spider/jianshu/load.py @@ -4,8 +4,8 @@ base = "D:\\Data\\" -def load_data(): - """ 加载数据 """ +def load_account(): + """ 加载账号 """ print("开始读取") data = csv.reader(open('{}data.csv'.format(base), encoding='utf-8')) @@ -31,5 +31,3 @@ def load_article(role): data.setdefault(article_name, article_content) print("读取结束".format(role)) return data - - diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 070c7bd..d600f56 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -24,7 +24,7 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) -logging.basicConfig(level='INFO', filename='info.log', +logging.basicConfig(level='INFO', filename='robot.log', format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') # 无头模式 options = Options() @@ -64,69 +64,37 @@ def sleep(min_seconds=3, max_seconds=10): page_title = driver.title if page_title.find("简书") == -1: logging.error("{} ：{}".format(role, page_title)) - - # 1. 收益 - driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) - elements = driver.find_elements_by_class_name("order") - info = ' ; '.join(map(lambda ele: ele.text, elements)) - logging.info("{} : {}".format(role, info)) - sleep() - - # 2.写文 - # driver.get("{}/writer#/".format(jianshu)), sleep() - # driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() - # - # if role == 0 or role == 1: - # title_prefix = "早晨" if is_day else "晚安" - # title_prefix2 = "每日一言" if role == 0 else "一言美句" - # driver.find_element_by_class_name("_24i7u").send_keys(Keys.HOME) - # driver.find_element_by_class_name("_24i7u").send_keys((title_prefix + "！" + title_prefix2 + "：")) - # else: - # resp = requests.get("https://v1.hitokoto.cn/").json() - # title = resp.get('hitokoto') - # driver.find_element_by_class_name("_24i7u").clear() - # driver.find_element_by_class_name("_24i7u").send_keys(title) - # - # content, editor, count = "", "arthur-editor", 0 - # while len(content) < 600: - # api = random.choice(('international.v1.hitokoto.cn', "v1.hitokoto.cn")) - # resp = requests.get("https://{}".format(api)).json() - # hitokoto = resp.get("hitokoto") - # if hitokoto: - # count += 1 - # content += hitokoto.strip() - # driver.find_element_by_id(editor).send_keys(str(count) + ". " + hitokoto) - # driver.find_element_by_id(editor).send_keys(Keys.ENTER) - # driver.find_element_by_id(editor).send_keys(Keys.ENTER) - # - # end = '> 动漫也好、小说也罢，不论在哪里，总会看到有那么一两个句子能穿透你的心，这就是一言。' - # driver.find_element_by_id(editor).send_keys(end) - # driver.find_element_by_css_selector("i[class='fa fa-mail-forward']").click() - # sleep() - - # 3.消息 - driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) - sleep() - - # 4.评论 - jianshu_p = "{}/p/".format(jianshu) - comments = {"ac02c56c0865": "38968576", "be27870bdba9": "38969657", "277f1b0a140d": "40692862", - "15eb212788fc": "40693328", "81d5ae85f38a": "40874580", "4dca23c2c75a": "40874520", - "9aaebad1753a": "42635258", "f3398c00ffc1": "42854890", "e369bb81bcc8": "42855033", - "4ec116446717": "42855114", "2d184d128522": "42855181", } - for comment_id, button_id in comments.items(): - driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)), sleep(3, 5) - button_element = 'like-button-{}'.format(button_id) - try: - driver.find_element_by_id(button_element).click(), sleep(3, 5) - driver.find_element_by_id(button_element).click() - except: - logging.error("Button_Element : {}".format(button_element)) - pass + else: + # 1. 收益 + driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) + elements = driver.find_elements_by_class_name("order") + info = ' ; '.join(map(lambda ele: ele.text, elements)) + logging.info("{} : {}".format(role, info)) + sleep() + + # 2.消息 + driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) + driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) + sleep() + + # 3.评论 + jianshu_p = "{}/p/".format(jianshu) + comments = {"ac02c56c0865": "38968576", "be27870bdba9": "38969657", "277f1b0a140d": "40692862", + "15eb212788fc": "40693328", "81d5ae85f38a": "40874580", "4dca23c2c75a": "40874520", + "9aaebad1753a": "42635258", "f3398c00ffc1": "42854890", "e369bb81bcc8": "42855033", + "4ec116446717": "42855114", "2d184d128522": "42855181", } + for comment_id, button_id in comments.items(): + driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)), sleep(3, 5) + button_element = 'like-button-{}'.format(button_id) + try: + driver.find_element_by_id(button_element).click(), sleep(3, 5) + driver.find_element_by_id(button_element).click() + except: + logging.error("Button_Element : {}".format(button_element)) + pass finally: driver.close() driver.quit() diff --git a/spider/jianshu/like.py b/spider/jianshu/sium.py similarity index 97% rename from spider/jianshu/like.py rename to spider/jianshu/sium.py index 295f7c1..4eb3c5b 100644 --- a/spider/jianshu/like.py +++ b/spider/jianshu/sium.py @@ -13,7 +13,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys -from . import data_ +from . import load logging.basicConfig(level='INFO', filename='like.log', format='%(message)s') headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} @@ -49,7 +49,7 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) -data = data_.load_data() +data = load.load_account() for row in data: username, password, role = row @@ -105,7 +105,7 @@ def sleep(min_seconds=3, max_seconds=10): logging.info("{}".format(url)) # 2. 写文 - article_data = data_.load_article(role) + article_data = load.load_article(role) driver.get("{}/writer#/".format(jianshu)), sleep() for title, content in article_data.items(): driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() diff --git a/spider/jianshu/spider.py b/spider/jianshu/spider.py deleted file mode 100644 index 1009bd0..0000000 --- a/spider/jianshu/spider.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import random -import sys -import time -from datetime import datetime - -import requests -from bs4 import BeautifulSoup -from sqlalchemy import Column, String, Integer, FLOAT, TIMESTAMP, BOOLEAN, create_engine -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker - -argv = sys.argv -if len(argv) > 1: - os.system("nohup python3 {} >> /dev/null 2>&1 &".format(argv[0])) -else: - # 基类 - base = declarative_base() - - - def get_now(): - return datetime.now() - - - class Article(base): - __tablename__ = "tb_jianshu_article" - - id = Column(Integer, primary_key=True, autoincrement=True) - article_url_suffix = Column(String(50), nullable=False, index=True, unique=True, comment="地址后缀") - ic_paid = Column(FLOAT(2), nullable=False, comment="简书钻数") - ic_like = Column(Integer, nullable=False, comment="喜欢人次") - ic_reward = Column(Integer, nullable=False, comment="赞赏人次") - comment_num = Column(Integer, nullable=False, comment="评论人次") - publish_time = Column(TIMESTAMP, nullable=False, comment="发布时间") - processed = Column(BOOLEAN, nullable=False, default=False, comment="处理状态。True:已处理;False:未处理") - update_time = Column(TIMESTAMP, nullable=False, comment="更新时间") - create_time = Column(TIMESTAMP, nullable=False, comment="创建时间") - - - conn = "postgresql+psycopg2://postgres:pgsmaster5432@localhost:12432/scrapy" - engine = create_engine(conn, encoding="UTF-8", echo=False) - base.metadata.create_all(engine) - - Session = sessionmaker(bind=engine) - session = Session() - - - def save_article(*args): - suffix, paid, like, num, p_time, reward = args - article = session.query(Article).filter(Article.article_url_suffix == suffix).first() - if not article: - # add - init_time = get_now() - article = Article(article_url_suffix=suffix, ic_paid=paid, ic_reward=reward, ic_like=like, - comment_num=num, publish_time=p_time, update_time=init_time, create_time=init_time) - else: - # update - article.ic_paid, article.ic_like, article.ic_reward, article.comment_num = paid, like, reward, num - article.update_time = get_now() - session.add(article) - session.commit() - - - domain = "https://www.jianshu.com" - headers = { - "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)" - } - # 我的关注列表 - following = "https://www.jianshu.com/users/000a530f461c/following?page={}" - - while True: - for page in range(1, 5): - resp = requests.get(following.format(page), headers=headers) - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, "html.parser") - user_list = bs.find("ul", class_="user-list") - avatar_list = user_list.find_all("a", class_="avatar") - - for avatar in avatar_list: - # 具体关注的人 - following_detail = domain + avatar["href"] - resp = requests.get(following_detail, headers=headers) - - if resp.status_code == 200: - html = resp.text - bs = BeautifulSoup(html, "html.parser") - content_list = bs.find_all("div", class_="content") - - for content in content_list: - meta = content.find("div", class_="meta") - - # 简书钻数 && 喜欢人次 && 发布时间 && 赞赏人次 - ic_paid = ic_like = ic_reward = 0 - publish_time = None - span_list = meta.find_all("span") - span_list_length = len(span_list) - if span_list_length == 2: - ic_like, publish_time = span_list[0].text.strip(), span_list[1]["data-shared-at"] - else: - ic_paid, ic_like = span_list[0].text.strip(), span_list[1].text.strip() - if span_list_length == 3: - publish_time = span_list[2]["data-shared-at"] - elif span_list_length == 4: - ic_reward, publish_time = span_list[2].text.strip(), span_list[3]["data-shared-at"] - publish_time = datetime.strptime(publish_time, "%Y-%m-%dT%H:%M:%S+08:00") - - publish_date = datetime.date(publish_time) - today_date = datetime.date(get_now()) - if publish_date == today_date: - # 查看次数 && 评论人次 - a_list = meta.find_all("a") - article_url_suffix, comment_num = a_list[0]["href"], a_list[1].text.strip() - - save_article(article_url_suffix, ic_paid, ic_like, comment_num, publish_time, ic_reward) - - print("sleep...") - time.sleep(random.randint(300, 600)) From 2f141fb2bec40bbcd816cbd4a1256a48190dc480 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 15:31:27 +0800 Subject: [PATCH 096/148] add log --- spider/jianshu/sium.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index 4eb3c5b..335b616 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -44,6 +44,8 @@ if count >= article_num: break +logging.info("共发布了{}篇文章".format(len(articles))) + def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) From 166550d2d101d22ce0ec597df8002a64a2b1d6f8 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 16:05:03 +0800 Subject: [PATCH 097/148] refactor jianshu --- spider/jianshu/aium.py | 24 ++++++++++++++++++++++-- spider/jianshu/{load.py => jsloader.py} | 6 +++--- spider/jianshu/sium.py | 6 +++--- 3 files changed, 28 insertions(+), 8 deletions(-) rename spider/jianshu/{load.py => jsloader.py} (83%) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 4374a9e..301c979 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,5 +1,5 @@ import time -from . import load +import jsloader from appium import webdriver from selenium.webdriver.common.by import By @@ -14,7 +14,7 @@ "appActivity": "com.baiji.jianshu.MainActivity" } -data = load.load_account() +data = jsloader.load_account() for row in data: username, password, role = row @@ -103,6 +103,26 @@ pass else: ele.click() + clickable = EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.view.View[2]')) + print("clickable {}".format(clickable)) # 根据元素是否还可以点击来判断是否还有抽奖次数 todo # 3. 关闭 ele = long_wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) diff --git a/spider/jianshu/load.py b/spider/jianshu/jsloader.py similarity index 83% rename from spider/jianshu/load.py rename to spider/jianshu/jsloader.py index 58609c1..ab73029 100644 --- a/spider/jianshu/load.py +++ b/spider/jianshu/jsloader.py @@ -9,9 +9,9 @@ def load_account(): print("开始读取") data = csv.reader(open('{}data.csv'.format(base), encoding='utf-8')) - for row in data: - username, password, role = row - print("Username: {} Password: {} Role: {}".format(username, password, role)) + # for row in data: + # username, password, role = row + # print("Username: {} Password: {} Role: {}".format(username, password, role)) print("读取结束") return data diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index 335b616..71bce3a 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -13,7 +13,7 @@ from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.keys import Keys -from . import load +import jsloader logging.basicConfig(level='INFO', filename='like.log', format='%(message)s') headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} @@ -51,7 +51,7 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) -data = load.load_account() +data = jsloader.load_account() for row in data: username, password, role = row @@ -107,7 +107,7 @@ def sleep(min_seconds=3, max_seconds=10): logging.info("{}".format(url)) # 2. 写文 - article_data = load.load_article(role) + article_data = jsloader.load_article(role) driver.get("{}/writer#/".format(jianshu)), sleep() for title, content in article_data.items(): driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() From 55992354f2d6685fbde1e30499ef9398fac6a3cc Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 16:38:16 +0800 Subject: [PATCH 098/148] refactor jianshu --- spider/jianshu/aium.py | 47 +++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 301c979..084b551 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,11 +1,12 @@ import time -import jsloader from appium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait +import jsloader + server = 'http://localhost:4723/wd/hub' desired_capabilities = { "platformName": "Android", @@ -16,6 +17,7 @@ data = jsloader.load_account() for row in data: + start = time.time() username, password, role = row driver = webdriver.Remote(server, desired_capabilities) @@ -78,31 +80,7 @@ 'android.view.View[2]/android.view.View[2]'))) ele.click() try: - # 2. 继续观看 - ele = short_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]'))) - - except: - pass - else: - ele.click() + # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) clickable = EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -122,14 +100,27 @@ 'android.view.View[2]/' 'android.view.View[2]/' 'android.view.View[2]')) - print("clickable {}".format(clickable)) - # 根据元素是否还可以点击来判断是否还有抽奖次数 todo + ele = short_wait.until(clickable) + except: + pass + else: + text = ele.text + print(text) + if text.find('今日已用完') != -1: + break + ele.click() # 3. 关闭 ele = long_wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() try: # 4. 知道了 todo + # ele = short_wait.until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, '知道了'))) time.sleep(5) driver.tap([(540, 1335)]) except: pass + else: + # ele.click() + pass + + print("{}用时{}秒".format(role, time.time() - start)) From 9239c4acd59d063583bf13821771f51196bc348f Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 16:42:45 +0800 Subject: [PATCH 099/148] refactor jianshu --- spider/jianshu/aium.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 084b551..4b0bf11 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -118,9 +118,29 @@ time.sleep(5) driver.tap([(540, 1335)]) except: - pass + # 奖励异常，后退重进 + driver.back() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() else: # ele.click() pass - print("{}用时{}秒".format(role, time.time() - start)) + print("Number{}.用时{}秒".format(role, int(time.time() - start))) From d65c6c692246fca89620849126166d044fbebe82 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 16:52:34 +0800 Subject: [PATCH 100/148] refactor jianshu --- spider/jianshu/aium.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 4b0bf11..ce87d1a 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -26,7 +26,7 @@ long_wait = WebDriverWait(driver, 60) # 我的 - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) ele.click() ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) ele.click() @@ -38,13 +38,13 @@ ele.send_keys(username) ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) ele.send_keys(password) - login = driver.find_element_by_id('com.jianshu.haruki:id/tv_login') - login.click() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) + ele.click() # 简书钻 -> 抽奖 - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' @@ -63,7 +63,7 @@ # 每日抽奖 while True: # 1. GO - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout' + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' '/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' @@ -110,17 +110,17 @@ break ele.click() # 3. 关闭 - ele = long_wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele = long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() try: # 4. 知道了 todo - # ele = short_wait.until(EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, '知道了'))) + # ele = short_wait.until(EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, '知道了'))) time.sleep(5) driver.tap([(540, 1335)]) except: # 奖励异常，后退重进 driver.back() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' From b63842b31e72cd26c261fda3fee81ef74652e1fe Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 17:05:17 +0800 Subject: [PATCH 101/148] refactor jianshu --- spider/jianshu/aium.py | 89 +++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index ce87d1a..73fa8da 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -19,6 +19,7 @@ for row in data: start = time.time() username, password, role = row + print("Number{}.".format(role)) driver = webdriver.Remote(server, desired_capabilities) wait = WebDriverWait(driver, 30) @@ -45,39 +46,39 @@ ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' - 'android.webkit.WebView/android.webkit.WebView/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[17]/android.view.View[3]'))) + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.webkit.WebView/android.webkit.WebView/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[17]/android.view.View[3]'))) ele.click() # 每日抽奖 while True: # 1. GO ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' - '/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + '/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() try: # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) @@ -121,23 +122,23 @@ # 奖励异常，后退重进 driver.back() ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() else: # ele.click() From 793618b2ddf4cad514b545e0b1ea8fc29ffcd043 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 17:16:08 +0800 Subject: [PATCH 102/148] refactor jianshu --- spider/jianshu/aium.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 73fa8da..f3b756b 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -145,3 +145,6 @@ pass print("Number{}.用时{}秒".format(role, int(time.time() - start))) + driver.close_app() + driver.close() + driver.quit() From 0536d4db0b048b25779b683150364b966cfd56e7 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 17:28:55 +0800 Subject: [PATCH 103/148] refactor jianshu --- spider/jianshu/aium.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index f3b756b..8a02d76 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -25,6 +25,7 @@ wait = WebDriverWait(driver, 30) short_wait = WebDriverWait(driver, 10) long_wait = WebDriverWait(driver, 60) + max_long_wait = WebDriverWait(driver, 100) # 我的 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) @@ -111,7 +112,7 @@ break ele.click() # 3. 关闭 - ele = long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele = max_long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() try: # 4. 知道了 todo From b0ce26f2ee542255ac4e1d360134971ca9e57109 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 18 Jul 2019 17:42:44 +0800 Subject: [PATCH 104/148] refactor jianshu --- spider/jianshu/jsloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index ab73029..45f3113 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -8,7 +8,7 @@ def load_account(): """ 加载账号 """ print("开始读取") - data = csv.reader(open('{}data.csv'.format(base), encoding='utf-8')) + data = csv.reader(open('{}data.csv'.format(base), "r", encoding='utf-8')) # for row in data: # username, password, role = row # print("Username: {} Password: {} Role: {}".format(username, password, role)) From 5523ee5041800e519533c60e65b7f4392a0f1621 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 00:10:58 +0800 Subject: [PATCH 105/148] update aium --- spider/jianshu/aium.py | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 8a02d76..cac77ca 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -115,13 +115,30 @@ ele = max_long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() try: - # 4. 知道了 todo - # ele = short_wait.until(EC.element_to_be_clickable((By.PARTIAL_LINK_TEXT, '知道了'))) - time.sleep(5) - driver.tap([(540, 1335)]) + # 4. 知道了 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) except: # 奖励异常，后退重进 - driver.back() + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -142,8 +159,7 @@ 'android.view.View[3]'))) ele.click() else: - # ele.click() - pass + ele.click() print("Number{}.用时{}秒".format(role, int(time.time() - start))) driver.close_app() From 9aec0a363f4cb02091b28fb159810f20f91c017f Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 09:44:01 +0800 Subject: [PATCH 106/148] refactor jianshu --- spider/jianshu/aium.py | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index cac77ca..57293eb 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -43,6 +43,14 @@ ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() + # 领钻弹框 + try: + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) + except: + pass + else: + ele.click() + # 简书钻 -> 抽奖 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() @@ -117,24 +125,24 @@ try: # 4. 知道了 ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.widget.Button'))) + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) except: # 奖励异常，后退重进 ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) From 52e9b9c29f74fefdf5bffb81fdca5ee0689dd90d Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 09:57:27 +0800 Subject: [PATCH 107/148] refactor jianshu --- spider/jianshu/aium.py | 72 ++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 57293eb..c77419b 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -51,26 +51,30 @@ else: ele.click() - # 简书钻 -> 抽奖 + # 简书钻 -> 天天抽奖 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' - 'android.webkit.WebView/android.webkit.WebView/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[17]/android.view.View[3]'))) + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() - # 每日抽奖 + # 抽奖 while True: # 1. GO ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' @@ -147,24 +151,24 @@ # 奖励异常，后退重进 ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) ele.click() - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() else: ele.click() From e2dfd85f41b2202aa470075ade5d923abe4a0ef6 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 10:47:56 +0800 Subject: [PATCH 108/148] refactor jianshu --- spider/jianshu/aium.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index c77419b..abe2bd2 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -77,7 +77,7 @@ # 抽奖 while True: # 1. GO - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' '/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' @@ -127,7 +127,7 @@ ele = max_long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() try: - # 4. 知道了 + # 4. 知道了 todo ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' From 60c544e752055fad28c308b3f92df9440ecbb7f8 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 10:53:50 +0800 Subject: [PATCH 109/148] refactor jianshu --- spider/jianshu/aium.py | 126 +++++++++++++++++++++-------------------- 1 file changed, 66 insertions(+), 60 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index abe2bd2..88d10a1 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -78,21 +78,23 @@ while True: # 1. GO ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' - '/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + '/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() + + keep_watch = False try: # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) clickable = EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' @@ -123,55 +125,59 @@ if text.find('今日已用完') != -1: break ele.click() + keep_watch = True + # 3. 关闭 ele = max_long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) ele.click() - try: - # 4. 知道了 todo - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.widget.Button'))) - except: - # 奖励异常，后退重进 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - else: - ele.click() + + # 4. 知道了 + if not keep_watch: + try: + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) + except: + # 奖励异常，后退重进 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() + else: + ele.click() print("Number{}.用时{}秒".format(role, int(time.time() - start))) driver.close_app() From a3d17181cf2381e844601794f1f9661e0a7bc3a3 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 10:54:15 +0800 Subject: [PATCH 110/148] refactor jianshu --- spider/jianshu/aium.py | 1 - 1 file changed, 1 deletion(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 88d10a1..6fb3357 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -180,6 +180,5 @@ ele.click() print("Number{}.用时{}秒".format(role, int(time.time() - start))) - driver.close_app() driver.close() driver.quit() From 14d0109030028a36bda2736f83b2c098d2512ca5 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 14:27:50 +0800 Subject: [PATCH 111/148] refactor jianshu --- spider/jianshu/aium.py | 141 ++++++++++++++++++++--------------------- 1 file changed, 69 insertions(+), 72 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 6fb3357..8cb4711 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -97,26 +97,25 @@ keep_watch = False try: # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) - clickable = EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]')) - ele = short_wait.until(clickable) + ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.view.View[2]'))) except: pass else: @@ -127,58 +126,56 @@ ele.click() keep_watch = True - # 3. 关闭 - ele = max_long_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - - # 4. 知道了 - if not keep_watch: - try: - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.widget.Button'))) - except: - # 奖励异常，后退重进 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - else: + try: + # 3. 关闭广告 + ele = max_long_wait.until( + EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + if not keep_watch: + # 4. 知道了 + ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) ele.click() + except: + print(" 奖励异常，后退重进") + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() print("Number{}.用时{}秒".format(role, int(time.time() - start))) - driver.close() - driver.quit() + driver.close_app() From 5d6f9120bb0677548434b644a415c1db4418fc70 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 17:29:38 +0800 Subject: [PATCH 112/148] refactor jianshu --- spider/jianshu/aium.py | 202 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 199 insertions(+), 3 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 8cb4711..b36aa5a 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,3 +1,4 @@ +import random import time from appium import webdriver @@ -18,7 +19,7 @@ data = jsloader.load_account() for row in data: start = time.time() - username, password, role = row + username, password, role, post_num = row print("Number{}.".format(role)) driver = webdriver.Remote(server, desired_capabilities) @@ -43,8 +44,8 @@ ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() - # 领钻弹框 try: + # 领钻弹框 ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) except: pass @@ -74,7 +75,7 @@ 'android.view.View[3]'))) ele.click() - # 抽奖 + # 一. 抽奖 while True: # 1. GO ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' @@ -178,4 +179,199 @@ ele.click() print("Number{}.用时{}秒".format(role, int(time.time() - start))) + + # 二. 发文 + driver.back() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + # 我的文章 + ele.click() + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.LinearLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.LinearLayout[2]/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout/' + 'android.widget.RelativeLayout[1]/' + 'android.widget.RelativeLayout'))) + ele.click() + # 私密文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' + 'android.widget.LinearLayout/' + 'android.support.v7.app.ActionBar.Tab[2]'))) + ele.click() + # 循环发布 + for i in range(0, post_num): + # 进入文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.RelativeLayout[1]'))) + ele.click() + # 公开发布 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) + ele.click() + # 两次返回 + driver.back() + driver.back() + + # 三. 浏览+收藏 + driver.back() + # 关注列表 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + ele.click() + + # 随机用户 + i = random.randint(1, 6) + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.RelativeLayout[{}]'.format(i)))) + ele.click() + + + def get_screen_size(): + """ 屏幕大小 """ + + x = driver.get_window_size()['width'] + y = driver.get_window_size()['height'] + return x, y + + + def swipe_up(t): + """ 屏幕上滑 """ + + screen_size = get_screen_size() + x1 = int(screen_size[0] * 0.5) # x坐标 + y1 = int(screen_size[1] * 0.75) # 起始y坐标 + y2 = int(screen_size[1] * 0.25) # 终点y坐标 + driver.swipe(x1, y1, x1, y2, t) + + + def swipe_down(t): + """ 屏幕下滑 """ + + screen_size = get_screen_size() + x1 = int(screen_size[0] * 0.5) # x坐标 + y1 = int(screen_size[1] * 0.25) # 起始y坐标 + y2 = int(screen_size[1] * 0.75) # 终点y坐标 + driver.swipe(x1, y1, x1, y2, t) + + + # 浏览文章 + for i in range(1, 20): + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.LinearLayout[{}]'.format(i)))) + ele.click() + + # 点击更多 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + if ele.text == '收藏': + ele.click() + # 下滑浏览 + for j in range(0, 10): + swipe_down(100) + time.sleep(1) + # 后退下滑 + driver.back() + swipe_down(100) + + # 四、签到 + driver.back() + driver.back() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) + ele.click() + # 任务领钻 + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[1]'))) + ele.click() + try: + # 每日签到 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.webkit.WebView/android.webkit.WebView/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[3]/android.view.View/' + 'android.view.View[1]'))) + except: + pass + else: + ele.click() + + # 五、退出APP driver.close_app() From ef6ea3fd49bcc0fdb0b299b58150eb1348cd97c0 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 19 Jul 2019 17:30:51 +0800 Subject: [PATCH 113/148] refactor jianshu --- spider/jianshu/sium.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index 71bce3a..db3823b 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -20,7 +20,8 @@ jianshu = "https://www.jianshu.com" articles = set() -user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", "b34dcc8c2394", "37ec6dd9fc1c", "04f7029dba14"] +user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", + "b34dcc8c2394", "37ec6dd9fc1c", "04f7029dba14", "32bbb0ecfcb9"] jianshu_u = "{}/u/".format(jianshu) for uid in user_ids: user = "{}{}".format(jianshu_u, uid) From 1c89bde7a18b9cc98c49e4a1a2dc901912da731b Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 20 Jul 2019 22:54:00 +0800 Subject: [PATCH 114/148] update jianshu --- spider/jianshu/aium.py | 434 ++++++++++++++++++++----------------- spider/jianshu/jsloader.py | 6 +- spider/jianshu/sium.py | 209 +++++++++++------- 3 files changed, 366 insertions(+), 283 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index b36aa5a..574caf2 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,4 +1,3 @@ -import random import time from appium import webdriver @@ -13,14 +12,18 @@ "platformName": "Android", "deviceName": "WAS_AL00", "appPackage": "com.jianshu.haruki", - "appActivity": "com.baiji.jianshu.MainActivity" + "appActivity": "com.baiji.jianshu.MainActivity", + 'unicodeKeyboard': True, # 是使用unicode编码方式发送字符串 + 'resetKeyboard': True, # 隐藏键盘 + 'noReset': True } -data = jsloader.load_account() +data = jsloader.load_account("jsdata") for row in data: start = time.time() username, password, role, post_num = row - print("Number{}.".format(role)) + post_num = int(post_num) + print("\nNumber{}.".format(role)) driver = webdriver.Remote(server, desired_capabilities) wait = WebDriverWait(driver, 30) @@ -43,6 +46,7 @@ ele.send_keys(password) ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() + print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) try: # 领钻弹框 @@ -52,88 +56,105 @@ else: ele.click() - # 简书钻 -> 天天抽奖 + # 简书钻 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - # 一. 抽奖 - while True: - # 1. GO - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout' - '/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + try: + """ 签到 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[1]'))) ele.click() - - keep_watch = False - try: - # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) - ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]'))) - except: - pass - else: - text = ele.text - print(text) - if text.find('今日已用完') != -1: - break + # 每日签到 + ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[1]'))) + except: + pass + else: + ele.click() + finally: + print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + # 后退 + time.sleep(2) + driver.back() + """ 抽奖 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() + while True: + # 1. GO + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() - keep_watch = True - try: - # 3. 关闭广告 - ele = max_long_wait.until( - EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - if not keep_watch: - # 4. 知道了 + keep_watch = False + try: + # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -148,39 +169,77 @@ 'android.widget.FrameLayout/' 'android.webkit.WebView/' 'android.webkit.WebView/' - 'android.view.View[1]/' + 'android.view.View/' 'android.view.View[1]/' 'android.view.View[2]/' 'android.view.View[2]/' - 'android.widget.Button'))) + 'android.view.View[2]'))) + except: + pass + else: + text = ele.text + print(text) + if text.find('今日已用完') != -1: + break ele.click() - except: - print(" 奖励异常，后退重进") - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() + keep_watch = True - print("Number{}.用时{}秒".format(role, int(time.time() - start))) + try: + # 3. 关闭广告 + ele = max_long_wait.until( + EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + if not keep_watch: + # 4. 知道了 + ele = short_wait.until( + EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + except: + print("奖励异常，后退重进") + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() - # 二. 发文 + print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + + """ 发文 """ + start = time.time() driver.back() ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) # 我的文章 @@ -233,35 +292,19 @@ 'android.support.v7.widget.RecyclerView/' 'android.widget.RelativeLayout[1]'))) ele.click() - # 公开发布 + # 发布 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) ele.click() - # 两次返回 - driver.back() + # 返回 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + ele.click() + # 点赞 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + ele.click() + # 返回 + time.sleep(3) driver.back() - - # 三. 浏览+收藏 - driver.back() - # 关注列表 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - ele.click() - - # 随机用户 - i = random.randint(1, 6) - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[{}]'.format(i)))) - ele.click() + print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) def get_screen_size(): @@ -292,86 +335,81 @@ def swipe_down(t): driver.swipe(x1, y1, x1, y2, t) - # 浏览文章 - for i in range(1, 20): + """ 关注 """ + start = time.time() + driver.back() + # 关注列表 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + ele.click() + for i in range(1, 7): ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' + 'android.widget.LinearLayout/' 'android.support.v4.view.ViewPager/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' 'android.view.ViewGroup/' 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[{}]'.format(i)))) + 'android.widget.RelativeLayout[{}]'.format(i)))) ele.click() - # 点击更多 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - if ele.text == '收藏': + # 浏览文章 + for j in range(1, 5): + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.LinearLayout[{}]'.format(j)))) ele.click() - # 下滑浏览 - for j in range(0, 10): - swipe_down(100) - time.sleep(1) - # 后退下滑 - driver.back() - swipe_down(100) - # 四、签到 - driver.back() - driver.back() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) - ele.click() - # 任务领钻 - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[1]'))) - ele.click() - try: - # 每日签到 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' - 'android.webkit.WebView/android.webkit.WebView/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[3]/android.view.View/' - 'android.view.View[1]'))) - except: - pass - else: + # 点赞文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + selected = ele.get_attribute("selected") + if selected == 'false': + ele.click() + swipe_down(30) + # 点击更多 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + if ele.text == '收藏': + ele.click() + else: + driver.tap([(18, 1235)]) + swipe_down(30) + # 评论文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + ele.click() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + ele.click() + ele.send_keys("不错不错，学习了") + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + ele.click() + # 后退下滑 + time.sleep(3) + driver.back() + time.sleep(3) + swipe_down(50) + + # 返回 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) ele.click() + print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) - # 五、退出APP + """ 退出APP """ driver.close_app() diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index 45f3113..2fb3a0c 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -1,14 +1,15 @@ import csv import os +from random import shuffle base = "D:\\Data\\" -def load_account(): +def load_account(file_name): """ 加载账号 """ print("开始读取") - data = csv.reader(open('{}data.csv'.format(base), "r", encoding='utf-8')) + data = csv.reader(open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8')) # for row in data: # username, password, role = row # print("Username: {} Password: {} Role: {}".format(username, password, role)) @@ -23,6 +24,7 @@ def load_article(role): path = "{}{}".format(base, role) file_list = os.listdir(path) print("文件夹路径【{}】下共有文章{}篇".format(path, len(file_list))) + shuffle(file_list) data = dict() for file_name in file_list: article_name = file_name.replace(".md", "") diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index db3823b..2a5852f 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -2,121 +2,164 @@ # @author : jared # @date : 2019/7/5 23:23 -import logging import random import time +from bs4 import BeautifulSoup import pyperclip import requests -from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait import jsloader -logging.basicConfig(level='INFO', filename='like.log', format='%(message)s') -headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} jianshu = "https://www.jianshu.com" - articles = set() -user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", - "b34dcc8c2394", "37ec6dd9fc1c", "04f7029dba14", "32bbb0ecfcb9"] -jianshu_u = "{}/u/".format(jianshu) -for uid in user_ids: - user = "{}{}".format(jianshu_u, uid) - html = requests.get(user, headers=headers).text - bs = BeautifulSoup(html, 'html.parser') - info = bs.find("div", class_="info") - p_list = info.find_all("p") - article_num = int(p_list[2].text) - - count = 0 - while count < article_num: - for page in range(1, 100): - html = requests.get("{}?order_by=shared_at&page={}".format(user, page), headers=headers).text - bs = BeautifulSoup(html, 'html.parser') - titles = bs.find_all('a', class_="title") - for title in titles: - article_url = title.get("href") - if article_url: - count += 1 - articles.add(article_url) - if count >= article_num: - break -logging.info("共发布了{}篇文章".format(len(articles))) + +def load_articles(): + """ 获取所有文章 """ + + if not articles: + start_load = time.time() + headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} + user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", "b34dcc8c2394", "37ec6dd9fc1c", + "04f7029dba14", "32bbb0ecfcb9"] + jianshu_u = "{}/u/".format(jianshu) + for uid in user_ids: + user = "{}{}".format(jianshu_u, uid) + html = requests.get(user, headers=headers).text + bs = BeautifulSoup(html, 'html.parser') + info = bs.find("div", class_="info") + p_list = info.find_all("p") + article_num = int(p_list[2].text) + + count = 0 + while count < article_num: + for page in range(1, 100): + html = requests.get("{}?order_by=shared_at&page={}".format(user, page), headers=headers).text + bs = BeautifulSoup(html, 'html.parser') + titles = bs.find_all('a', class_="title") + for article_title in titles: + article_url = article_title.get("href") + if article_url: + count += 1 + articles.add(article_url) + if count >= article_num: + break + + print("共发布了{}篇".format(len(articles))) + print("获取用时{}秒".format(int(time.time() - start_load))) + return articles def sleep(min_seconds=3, max_seconds=10): + """ 随机休眠 """ + time.sleep(random.randint(min_seconds, max_seconds)) -data = jsloader.load_account() +data = jsloader.load_account("dbdata") for row in data: - - username, password, role = row + username, password, role, post_num, like = row + post_num = int(post_num) + if post_num <= 0 and like != "T": + continue + start = time.time() options = Options() - if role != "6": - options.add_argument('--headless') - options.add_argument('log-level=3') - options.add_argument('--no-sandbox') - options.add_argument('--disable-gpu') - options.add_argument('window-size=1366x728') - options.add_argument('--disable-dev-shm-usage') - options.add_argument('blink-settings=imagesEnabled=false') + # if role != "6": + # options.add_argument('--headless') + # options.add_argument('log-level=3') + # options.add_argument('--no-sandbox') + # options.add_argument('--disable-gpu') + # options.add_argument('window-size=1366x728') + # options.add_argument('--disable-dev-shm-usage') + # options.add_argument('blink-settings=imagesEnabled=false') driver = webdriver.Chrome(chrome_options=options) + wait = WebDriverWait(driver, 30) + short_wait = WebDriverWait(driver, 10) + long_wait = WebDriverWait(driver, 60) + max_long_wait = WebDriverWait(driver, 100) driver.get("{}/sign_in".format(jianshu)) driver.maximize_window() - if role != "6": - driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() - driver.find_element_by_class_name("douban").click() - sleep() - - driver.close() - window_handles = driver.window_handles - driver.switch_to.window(window_handles[-1]) - driver.find_element_by_id("inp-alias").send_keys(username) - driver.find_element_by_id("inp-pwd").send_keys(password) - try: + # 登录 if role != "6": - driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() - except: - logging.error("{} : 授权登录失败".format(role)) - else: - if role != "6": - sleep() + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div[2]/div/div/ul/li[4]'))) + ele.click() + ele = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'douban'))) + ele.click() + + sleep(3, 5) + driver.close() + window_handles = driver.window_handles + driver.switch_to.window(window_handles[-1]) + + username_ele = wait.until(EC.element_to_be_clickable((By.ID, 'inp-alias'))) + password_ele = wait.until(EC.element_to_be_clickable((By.ID, 'inp-pwd'))) + login_ele = wait.until( + EC.element_to_be_clickable((By.XPATH, '/html/body/div/div[2]/form[2]/div[4]/input[1]'))) else: - sleep(45, 60) + username_ele = wait.until(EC.element_to_be_clickable((By.ID, 'session_email_or_mobile_number'))) + password_ele = wait.until(EC.element_to_be_clickable((By.ID, 'session_password'))) + login_ele = wait.until(EC.element_to_be_clickable((By.ID, 'sign-in-form-submit-btn'))) - page_title = driver.title - if page_title.find("简书") == -1: - logging.error("{} ：{} 登录失败".format(role, page_title)) + username_ele.send_keys(username) + password_ele.send_keys(password) + login_ele.click() + except: + print("{} : 登录失败".format(role)) + else: + try: + # 通过该元素判断是否正常登录 + menu_ele = wait.until(EC.element_to_be_clickable((By.ID, 'menu'))) + except: + print("{} ：{} 登录失败".format(role, driver.title)) else: - logging.info("Role: {}".format(role)) - - # 1. 喜欢 - for article in articles: - url = "{}{}".format(jianshu, article) - driver.get(url) - bs = BeautifulSoup(driver.page_source, 'html.parser') - like_element = bs.find("div", class_="btn like-group active") - if not like_element: - logging.info("{}".format(url)) - - # 2. 写文 - article_data = jsloader.load_article(role) - driver.get("{}/writer#/".format(jianshu)), sleep() - for title, content in article_data.items(): - driver.find_element_by_css_selector("i[class='fa fa-plus-circle']").click(), sleep() - driver.find_element_by_class_name("_24i7u").clear() - driver.find_element_by_class_name("_24i7u").send_keys(title), sleep() - driver.find_element_by_id("arthur-editor").clear() - pyperclip.copy(content) - driver.find_element_by_id("arthur-editor").send_keys(Keys.CONTROL, 'v'), sleep() + print("\nNumber{}.".format(role)) + print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) + + """ 写文 """ + if post_num > 0: + start = time.time() + article_data = jsloader.load_article(role) + driver.get("{}/writer#/".format(jianshu)) + sleep() + for title, content in article_data.items(): + ele = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "i[class='fa fa-plus-circle']"))) + ele.click() + sleep(2, 3) + + ele = wait.until(EC.presence_of_element_located((By.CLASS_NAME, "_24i7u"))) + ele.clear() + ele.send_keys(title) + sleep(2, 3) + + pyperclip.copy(content) + ele = wait.until(EC.presence_of_element_located((By.ID, "arthur-editor"))) + ele.clear() + ele.send_keys(Keys.CONTROL, 'v') + sleep(2, 3) + print("Number{}.写文用时{}秒".format(role, int(time.time() - start))) + + """ 喜欢 """ + if like == "T": + load_articles() + start = time.time() + for article in articles: + url = "{}{}".format(jianshu, article) + driver.get(url) + beautiful_soup = BeautifulSoup(driver.page_source, 'html.parser') + like_element = beautiful_soup.find("div", class_="btn like-group active") + if not like_element: + print("{}".format(url)) + print("Number{}.喜欢用时{}秒".format(role, int(time.time() - start))) finally: driver.close() driver.quit() From 7be93c8681677074e8826338cee9f13d57e5e177 Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 20 Jul 2019 23:22:04 +0800 Subject: [PATCH 115/148] update jianshu --- spider/jianshu/aium.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 574caf2..f7aa62f 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,4 +1,5 @@ import time +import random from appium import webdriver from selenium.webdriver.common.by import By @@ -320,7 +321,7 @@ def swipe_up(t): screen_size = get_screen_size() x1 = int(screen_size[0] * 0.5) # x坐标 - y1 = int(screen_size[1] * 0.75) # 起始y坐标 + y1 = int(screen_size[1] * 0.4) # 起始y坐标 y2 = int(screen_size[1] * 0.25) # 终点y坐标 driver.swipe(x1, y1, x1, y2, t) @@ -331,7 +332,7 @@ def swipe_down(t): screen_size = get_screen_size() x1 = int(screen_size[0] * 0.5) # x坐标 y1 = int(screen_size[1] * 0.25) # 起始y坐标 - y2 = int(screen_size[1] * 0.75) # 终点y坐标 + y2 = int(screen_size[1] * 0.4) # 终点y坐标 driver.swipe(x1, y1, x1, y2, t) @@ -381,7 +382,7 @@ def swipe_down(t): selected = ele.get_attribute("selected") if selected == 'false': ele.click() - swipe_down(30) + swipe_up(random.randint(50, 100)) # 点击更多 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) ele.click() @@ -391,7 +392,7 @@ def swipe_down(t): ele.click() else: driver.tap([(18, 1235)]) - swipe_down(30) + swipe_up(random.randint(50, 100)) # 评论文章 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) ele.click() @@ -401,10 +402,10 @@ def swipe_down(t): ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) ele.click() # 后退下滑 - time.sleep(3) + time.sleep(random.randint(2, 3)) driver.back() - time.sleep(3) - swipe_down(50) + time.sleep(random.randint(2, 3)) + swipe_up(random.randint(600, 800)) # 返回 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) From d40bac29c6a9ded4397130ce2a3e634d6c69c1e9 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 21 Jul 2019 12:07:19 +0800 Subject: [PATCH 116/148] update jianshu --- spider/jianshu/aium.py | 405 +++++++++++++++++++++++++++---------- spider/jianshu/jsloader.py | 15 ++ 2 files changed, 313 insertions(+), 107 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index f7aa62f..5e6b7a0 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,5 +1,5 @@ import time -import random +from math import floor from appium import webdriver from selenium.webdriver.common.by import By @@ -16,10 +16,11 @@ "appActivity": "com.baiji.jianshu.MainActivity", 'unicodeKeyboard': True, # 是使用unicode编码方式发送字符串 'resetKeyboard': True, # 隐藏键盘 - 'noReset': True + # 'noReset': True } data = jsloader.load_account("jsdata") +comment_data = jsloader.load_comment() for row in data: start = time.time() username, password, role, post_num = row @@ -60,7 +61,26 @@ # 简书钻 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() - + # 持有借钻 + time.sleep(5) + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[15]'))) + hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) try: """ 签到 """ start = time.time() @@ -109,9 +129,182 @@ ele.click() finally: print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + + """ 领钻 """ + start = time.time() + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.view.View[1]'))) + receive = 0 + jz_num = int(ele.text) + while hole_jz_num < 5000 and jz_num > 0 and receive < 10: + # 领取 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.widget.Button[1]'))) + ele.click() + # 完成 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.view.View[1]'))) + jz_num = int(ele.text) + receive += 1 + print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) + # 后退 time.sleep(2) driver.back() + + """ 贝转钻 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[10]'))) + jsb_num = floor(float(ele.text)) + if jsb_num > 0: + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[12]'))) + ele.click() + # 输入数量 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.widget.EditText'))) + ele.click() + ele.send_keys(jsb_num) + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.widget.Button'))) + ele.click() + # 确认转换 + print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) + """ 抽奖 """ start = time.time() ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' @@ -302,115 +495,113 @@ # 点赞 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) ele.click() + time.sleep(3) + # 点击更多 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + ele.click() # 返回 time.sleep(3) driver.back() print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - - def get_screen_size(): - """ 屏幕大小 """ - - x = driver.get_window_size()['width'] - y = driver.get_window_size()['height'] - return x, y - - - def swipe_up(t): - """ 屏幕上滑 """ - - screen_size = get_screen_size() - x1 = int(screen_size[0] * 0.5) # x坐标 - y1 = int(screen_size[1] * 0.4) # 起始y坐标 - y2 = int(screen_size[1] * 0.25) # 终点y坐标 - driver.swipe(x1, y1, x1, y2, t) - - - def swipe_down(t): - """ 屏幕下滑 """ - - screen_size = get_screen_size() - x1 = int(screen_size[0] * 0.5) # x坐标 - y1 = int(screen_size[1] * 0.25) # 起始y坐标 - y2 = int(screen_size[1] * 0.4) # 终点y坐标 - driver.swipe(x1, y1, x1, y2, t) - - - """ 关注 """ - start = time.time() - driver.back() - # 关注列表 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - ele.click() - for i in range(1, 7): - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[{}]'.format(i)))) - ele.click() - - # 浏览文章 - for j in range(1, 5): - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[{}]'.format(j)))) - ele.click() - - # 点赞文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - selected = ele.get_attribute("selected") - if selected == 'false': - ele.click() - swipe_up(random.randint(50, 100)) - # 点击更多 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - if ele.text == '收藏': - ele.click() - else: - driver.tap([(18, 1235)]) - swipe_up(random.randint(50, 100)) - # 评论文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - ele.click() - ele.send_keys("不错不错，学习了") - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - ele.click() - # 后退下滑 - time.sleep(random.randint(2, 3)) - driver.back() - time.sleep(random.randint(2, 3)) - swipe_up(random.randint(600, 800)) - - # 返回 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - ele.click() - print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + # def get_screen_size(): + # """ 屏幕大小 """ + # + # x = driver.get_window_size()['width'] + # y = driver.get_window_size()['height'] + # return x, y + # + # + # def swipe_up(t): + # """ 屏幕上滑 """ + # + # screen_size = get_screen_size() + # x1 = int(screen_size[0] * 0.5) # x坐标 + # y1 = int(screen_size[1] * 0.5) # 起始y坐标 + # y2 = int(screen_size[1] * 0.25) # 终点y坐标 + # driver.swipe(x1, y1, x1, y2, t) + # + # + # """ 关注 """ + # start = time.time() + # driver.back() + # # 关注列表 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + # ele.click() + # for i in range(1, 7): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.RelativeLayout[{}]'.format(i)))) + # ele.click() + # + # swipe_up(random.randint(600, 800)) + # + # # 浏览文章 + # for j in range(1, 5): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.LinearLayout[{}]'.format(j)))) + # ele.click() + # + # # 点赞文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + # selected = ele.get_attribute("selected") + # if selected == 'false': + # ele.click() + # swipe_up(50) + # # 点击更多 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # ele.click() + # # 收藏文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # if ele.text == '收藏': + # ele.click() + # else: + # driver.tap([(18, 1235)]) + # swipe_up(50) + # # 评论文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + # ele.click() + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + # ele.click() + # ele.send_keys(choice(comment_data)) + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + # ele.click() + # # 后退下滑 + # time.sleep(random.randint(2, 3)) + # driver.back() + # time.sleep(random.randint(2, 3)) + # + # # 返回 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) + # ele.click() + # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) """ 退出APP """ driver.close_app() + driver.quit() diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index 2fb3a0c..a9627f8 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -33,3 +33,18 @@ def load_article(role): data.setdefault(article_name, article_content) print("读取结束".format(role)) return data + + +def load_comment(): + """ 加载评论 """ + + print("开始读取") + data = list() + reader = csv.reader(open('{}{}.csv'.format(base, "comment"), "r", encoding='utf-8')) + for row in reader: + data.append(row[0]) + shuffle(data) + print("读取结束") + return data + + From 3bd4018d284c5b05ce92313c56fb890ddb251026 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 21 Jul 2019 12:13:48 +0800 Subject: [PATCH 117/148] update jianshu --- spider/jianshu/aium.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 5e6b7a0..c499bb2 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -80,7 +80,9 @@ 'android.view.View[1]/' 'android.view.View[1]/' 'android.view.View[15]'))) - hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) + # hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) + print(ele.text) + hole_jz_num = 0 try: """ 签到 """ start = time.time() From e79deaa7166bbda75cd4944c401a537827f0903d Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 22 Jul 2019 11:06:53 +0800 Subject: [PATCH 118/148] refactor jianshu --- spider/jianshu/aium.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index c499bb2..4f2ef1b 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -80,9 +80,10 @@ 'android.view.View[1]/' 'android.view.View[1]/' 'android.view.View[15]'))) - # hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - print(ele.text) hole_jz_num = 0 + if ele.text.find("持有借钻") != -1: + hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) + print(ele.text, " ", hole_jz_num) try: """ 签到 """ start = time.time() @@ -497,7 +498,7 @@ # 点赞 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) ele.click() - time.sleep(3) + time.sleep(4) # 点击更多 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) ele.click() @@ -505,7 +506,7 @@ ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) ele.click() # 返回 - time.sleep(3) + time.sleep(2) driver.back() print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) From 8c1ad10a837cdc6fd3dbb22279f68027e0b5bbeb Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 22 Jul 2019 12:43:23 +0800 Subject: [PATCH 119/148] refactor jianshu --- spider/jianshu/aium.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 4f2ef1b..5dc5f62 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -25,6 +25,8 @@ start = time.time() username, password, role, post_num = row post_num = int(post_num) + if post_num <= 0: + break print("\nNumber{}.".format(role)) driver = webdriver.Remote(server, desired_capabilities) @@ -492,13 +494,24 @@ # 发布 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) ele.click() + # 点赞 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.ScrollView/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout[1]/' + 'android.widget.FrameLayout[3]/' + 'android.widget.RelativeLayout/' + 'android.widget.FrameLayout/android.widget.ImageView'))) + ele.click() + time.sleep(2) # 返回 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) ele.click() - # 点赞 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - ele.click() - time.sleep(4) # 点击更多 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) ele.click() From 2ab720215e1bd0ecdd961124f79e646a2ddfd0ee Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 22 Jul 2019 17:37:06 +0800 Subject: [PATCH 120/148] refactor jianshu --- spider/jianshu/aium.py | 212 +++++++++++++++++++++-------------------- 1 file changed, 108 insertions(+), 104 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 5dc5f62..224c1df 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,7 +1,8 @@ +import random import time -from math import floor from appium import webdriver +from math import floor from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait @@ -16,7 +17,7 @@ "appActivity": "com.baiji.jianshu.MainActivity", 'unicodeKeyboard': True, # 是使用unicode编码方式发送字符串 'resetKeyboard': True, # 隐藏键盘 - # 'noReset': True + # 'noReset': True # 不清空会话 } data = jsloader.load_account("jsdata") @@ -85,7 +86,7 @@ hole_jz_num = 0 if ele.text.find("持有借钻") != -1: hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - print(ele.text, " ", hole_jz_num) + print(ele.text, "/", hole_jz_num) try: """ 签到 """ start = time.time() @@ -508,115 +509,118 @@ 'android.widget.RelativeLayout/' 'android.widget.FrameLayout/android.widget.ImageView'))) ele.click() - time.sleep(2) # 返回 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) ele.click() - # 点击更多 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - ele.click() + try: + # 点击更多 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + ele.click() + except: + pass # 返回 time.sleep(2) driver.back() print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - # def get_screen_size(): - # """ 屏幕大小 """ - # - # x = driver.get_window_size()['width'] - # y = driver.get_window_size()['height'] - # return x, y - # - # - # def swipe_up(t): - # """ 屏幕上滑 """ - # - # screen_size = get_screen_size() - # x1 = int(screen_size[0] * 0.5) # x坐标 - # y1 = int(screen_size[1] * 0.5) # 起始y坐标 - # y2 = int(screen_size[1] * 0.25) # 终点y坐标 - # driver.swipe(x1, y1, x1, y2, t) - # - # - # """ 关注 """ - # start = time.time() - # driver.back() - # # 关注列表 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - # ele.click() - # for i in range(1, 7): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.RelativeLayout[{}]'.format(i)))) - # ele.click() - # - # swipe_up(random.randint(600, 800)) - # - # # 浏览文章 - # for j in range(1, 5): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.LinearLayout[{}]'.format(j)))) - # ele.click() - # - # # 点赞文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - # selected = ele.get_attribute("selected") - # if selected == 'false': - # ele.click() - # swipe_up(50) - # # 点击更多 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - # ele.click() - # # 收藏文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - # if ele.text == '收藏': - # ele.click() - # else: - # driver.tap([(18, 1235)]) - # swipe_up(50) - # # 评论文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - # ele.click() - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - # ele.click() - # ele.send_keys(choice(comment_data)) - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - # ele.click() - # # 后退下滑 - # time.sleep(random.randint(2, 3)) - # driver.back() - # time.sleep(random.randint(2, 3)) - # - # # 返回 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - # ele.click() - # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + + def get_screen_size(): + """ 屏幕大小 """ + + x = driver.get_window_size()['width'] + y = driver.get_window_size()['height'] + return x, y + + + def swipe_up(t): + """ 向上滑动，屏幕下滑 """ + + screen_size = get_screen_size() + x1 = int(screen_size[0] * 0.5) # x坐标 + y1 = int(screen_size[1] * 0.6) # 起始y坐标 + y2 = int(screen_size[1] * 0.25) # 终点y坐标 + driver.swipe(x1, y1, x1, y2, t) + + + """ 关注 """ + start = time.time() + driver.back() + # 关注列表 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + ele.click() + for i in range(1, 7): + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.RelativeLayout[{}]'.format(i)))) + ele.click() + + swipe_up(random.randint(600, 800)) + + # 浏览文章 + for j in range(1, 5): + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.view.ViewGroup/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.LinearLayout[{}]'.format(j)))) + ele.click() + + # 点击更多 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + if ele.text == '收藏': + ele.click() + else: + driver.tap([(18, 1235)]) + swipe_up(800) + # 点赞文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + selected = ele.get_attribute("selected") + if selected == 'false': + ele.click() + swipe_up(800) + # 评论文章 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + ele.click() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + ele.click() + ele.send_keys(random.choice(comment_data)) + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + ele.click() + # 后退下滑 + time.sleep(random.randint(2, 3)) + driver.back() + time.sleep(random.randint(2, 3)) + + # 返回 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) + ele.click() + print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) """ 退出APP """ driver.close_app() From 4a2237243ee5465eacf09adc1acf88241328095f Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 23 Jul 2019 13:34:51 +0800 Subject: [PATCH 121/148] refactor jianshu --- spider/jianshu/aium.py | 197 ++++++++++++++++++++--------------------- 1 file changed, 98 insertions(+), 99 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 224c1df..2ba931f 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -509,10 +509,10 @@ 'android.widget.RelativeLayout/' 'android.widget.FrameLayout/android.widget.ImageView'))) ele.click() - # 返回 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) - ele.click() try: + # 返回 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + ele.click() # 点击更多 ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) ele.click() @@ -520,107 +520,106 @@ ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) ele.click() except: - pass + driver.back() # 返回 time.sleep(2) driver.back() print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - - def get_screen_size(): - """ 屏幕大小 """ - - x = driver.get_window_size()['width'] - y = driver.get_window_size()['height'] - return x, y - - - def swipe_up(t): - """ 向上滑动，屏幕下滑 """ - - screen_size = get_screen_size() - x1 = int(screen_size[0] * 0.5) # x坐标 - y1 = int(screen_size[1] * 0.6) # 起始y坐标 - y2 = int(screen_size[1] * 0.25) # 终点y坐标 - driver.swipe(x1, y1, x1, y2, t) - - - """ 关注 """ - start = time.time() - driver.back() - # 关注列表 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - ele.click() - for i in range(1, 7): - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[{}]'.format(i)))) - ele.click() - - swipe_up(random.randint(600, 800)) - - # 浏览文章 - for j in range(1, 5): - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[{}]'.format(j)))) - ele.click() - - # 点击更多 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - if ele.text == '收藏': - ele.click() - else: - driver.tap([(18, 1235)]) - swipe_up(800) - # 点赞文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - selected = ele.get_attribute("selected") - if selected == 'false': - ele.click() - swipe_up(800) - # 评论文章 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - ele.click() - ele.send_keys(random.choice(comment_data)) - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - ele.click() - # 后退下滑 - time.sleep(random.randint(2, 3)) - driver.back() - time.sleep(random.randint(2, 3)) - - # 返回 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - ele.click() - print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + # def get_screen_size(): + # """ 屏幕大小 """ + # + # x = driver.get_window_size()['width'] + # y = driver.get_window_size()['height'] + # return x, y + # + # + # def swipe_up(t): + # """ 向上滑动，屏幕下滑 """ + # + # screen_size = get_screen_size() + # x1 = int(screen_size[0] * 0.5) # x坐标 + # y1 = int(screen_size[1] * 0.6) # 起始y坐标 + # y2 = int(screen_size[1] * 0.25) # 终点y坐标 + # driver.swipe(x1, y1, x1, y2, t) + # + # + # """ 关注 """ + # start = time.time() + # driver.back() + # # 关注列表 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + # ele.click() + # for i in range(1, 7): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.RelativeLayout[{}]'.format(i)))) + # ele.click() + # + # swipe_up(random.randint(600, 800)) + # + # # 浏览文章 + # for j in range(1, 5): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.LinearLayout[{}]'.format(j)))) + # ele.click() + # + # # 点击更多 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # ele.click() + # # 收藏文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # if ele.text == '收藏': + # ele.click() + # else: + # driver.tap([(18, 1235)]) + # swipe_up(800) + # # 点赞文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + # selected = ele.get_attribute("selected") + # if selected == 'false': + # ele.click() + # swipe_up(800) + # # 评论文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + # ele.click() + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + # ele.click() + # ele.send_keys(random.choice(comment_data)) + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + # ele.click() + # # 后退下滑 + # time.sleep(random.randint(2, 3)) + # driver.back() + # time.sleep(random.randint(2, 3)) + # + # # 返回 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) + # ele.click() + # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) """ 退出APP """ driver.close_app() From 74547c8a6820a46fd83f6324a6174855eaa1ba72 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 24 Jul 2019 10:34:10 +0800 Subject: [PATCH 122/148] refactor jianshu --- spider/jianshu/aium.py | 1018 ++++++++++++++++++++-------------------- 1 file changed, 512 insertions(+), 506 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 2ba931f..7a773f6 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -30,122 +30,52 @@ break print("\nNumber{}.".format(role)) - driver = webdriver.Remote(server, desired_capabilities) - wait = WebDriverWait(driver, 30) - short_wait = WebDriverWait(driver, 10) - long_wait = WebDriverWait(driver, 60) - max_long_wait = WebDriverWait(driver, 100) - - # 我的 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) - ele.click() - - # 登录 - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) - ele.send_keys(username) - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) - ele.send_keys(password) - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) - ele.click() - print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) - try: - # 领钻弹框 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) - except: - pass - else: + driver = webdriver.Remote(server, desired_capabilities) + wait = WebDriverWait(driver, 30) + short_wait = WebDriverWait(driver, 10) + long_wait = WebDriverWait(driver, 60) + max_long_wait = WebDriverWait(driver, 100) + + # 我的 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) ele.click() - # 简书钻 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) - ele.click() - # 持有借钻 - time.sleep(5) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[15]'))) - hole_jz_num = 0 - if ele.text.find("持有借钻") != -1: - hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - print(ele.text, "/", hole_jz_num) - try: - """ 签到 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[1]'))) + # 登录 + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) ele.click() - # 每日签到 - ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[1]'))) - except: - pass - else: + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) + ele.send_keys(username) + ele = wait.until( + EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) + ele.send_keys(password) + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() - finally: - print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) - """ 领钻 """ - start = time.time() + try: + # 领钻弹框 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) + except: + pass + else: + ele.click() + + # 简书钻 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) + ele.click() + # 持有借钻 + time.sleep(5) ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' 'android.widget.FrameLayout/' 'android.widget.RelativeLayout/' 'android.view.ViewGroup/' @@ -154,55 +84,62 @@ 'android.webkit.WebView/' 'android.view.View[1]/' 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.view.View[1]'))) - receive = 0 - jz_num = int(ele.text) - while hole_jz_num < 5000 and jz_num > 0 and receive < 10: - # 领取 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.widget.Button[1]'))) + 'android.view.View[15]'))) + hole_jz_num = 0 + if ele.text.find("持有借钻") != -1: + hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) + print(ele.text, "/", hole_jz_num) + try: + """ 签到 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[1]'))) ele.click() - # 完成 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[2]/' - 'android.widget.Button'))) + # 每日签到 + ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[1]'))) + except: + pass + else: ele.click() + finally: + print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + + """ 领钻 """ + start = time.time() ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -222,175 +159,204 @@ 'android.view.View[2]/' 'android.view.View[5]/' 'android.view.View[1]'))) + receive = 0 jz_num = int(ele.text) - receive += 1 - print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) + while hole_jz_num < 5000 and jz_num > 0 and receive < 10: + # 领取 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.widget.Button[1]'))) + ele.click() + # 完成 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.view.View[1]'))) + jz_num = int(ele.text) + receive += 1 + print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) - # 后退 - time.sleep(2) - driver.back() + # 后退 + time.sleep(2) + driver.back() - """ 贝转钻 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[10]'))) - jsb_num = floor(float(ele.text)) - if jsb_num > 0: - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[12]'))) - ele.click() - # 输入数量 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.widget.EditText'))) - ele.click() - ele.send_keys(jsb_num) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.widget.Button'))) - ele.click() - # 确认转换 - print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) + """ 贝转钻 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[10]'))) + jsb_num = floor(float(ele.text)) + if jsb_num > 0: + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[12]'))) + ele.click() + # 输入数量 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.widget.EditText'))) + ele.click() + ele.send_keys(jsb_num) + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.widget.Button'))) + ele.click() + # 确认转换 + print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) - """ 抽奖 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - while True: - # 1. GO - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + """ 抽奖 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() - - keep_watch = False - try: - # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) - ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]'))) - except: - pass - else: - text = ele.text - print(text) - if text.find('今日已用完') != -1: - break + while True: + # 1. GO + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() - keep_watch = True - try: - # 3. 关闭广告 - ele = max_long_wait.until( - EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - if not keep_watch: - # 4. 知道了 + keep_watch = False + try: + # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) ele = short_wait.until( EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -406,75 +372,82 @@ 'android.widget.FrameLayout/' 'android.webkit.WebView/' 'android.webkit.WebView/' - 'android.view.View[1]/' + 'android.view.View/' 'android.view.View[1]/' 'android.view.View[2]/' 'android.view.View[2]/' - 'android.widget.Button'))) + 'android.view.View[2]'))) + except: + pass + else: + text = ele.text + print(text) + if text.find('今日已用完') != -1: + break ele.click() - except: - print("奖励异常，后退重进") - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() + keep_watch = True - print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + try: + # 3. 关闭广告 + ele = max_long_wait.until( + EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + if not keep_watch: + # 4. 知道了 + ele = short_wait.until( + EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + except: + print("奖励异常，后退重进") + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + ele = long_wait.until( + EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() - """ 发文 """ - start = time.time() - driver.back() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - # 我的文章 - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.LinearLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[2]/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout/' - 'android.widget.RelativeLayout[1]/' - 'android.widget.RelativeLayout'))) - ele.click() - # 私密文章 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' - 'android.widget.LinearLayout/' - 'android.support.v7.app.ActionBar.Tab[2]'))) - ele.click() - # 循环发布 - for i in range(0, post_num): - # 进入文章 + print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + + """ 发文 """ + start = time.time() + driver.back() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + # 我的文章 + ele.click() ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -482,145 +455,178 @@ 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.view.ViewGroup/' 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[1]'))) - ele.click() - # 发布 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) + 'android.widget.LinearLayout[2]/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout/' + 'android.widget.RelativeLayout[1]/' + 'android.widget.RelativeLayout'))) ele.click() - # 点赞 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.ScrollView/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[3]/' - 'android.widget.RelativeLayout/' - 'android.widget.FrameLayout/android.widget.ImageView'))) + # 私密文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' + 'android.widget.LinearLayout/' + 'android.support.v7.app.ActionBar.Tab[2]'))) ele.click() - try: - # 返回 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + # 循环发布 + for i in range(0, post_num): + # 进入文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.RelativeLayout[1]'))) ele.click() - # 点击更多 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # 发布 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) ele.click() - # 收藏文章 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # 点赞 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.ScrollView/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout[1]/' + 'android.widget.FrameLayout[3]/' + 'android.widget.RelativeLayout/' + 'android.widget.FrameLayout/android.widget.ImageView'))) ele.click() - except: + try: + # 返回 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + ele.click() + # 点击更多 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + ele.click() + except: + driver.back() + # 返回 + time.sleep(2) driver.back() - # 返回 - time.sleep(2) - driver.back() - print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) + print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - # def get_screen_size(): - # """ 屏幕大小 """ - # - # x = driver.get_window_size()['width'] - # y = driver.get_window_size()['height'] - # return x, y - # - # - # def swipe_up(t): - # """ 向上滑动，屏幕下滑 """ - # - # screen_size = get_screen_size() - # x1 = int(screen_size[0] * 0.5) # x坐标 - # y1 = int(screen_size[1] * 0.6) # 起始y坐标 - # y2 = int(screen_size[1] * 0.25) # 终点y坐标 - # driver.swipe(x1, y1, x1, y2, t) - # - # - # """ 关注 """ - # start = time.time() - # driver.back() - # # 关注列表 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - # ele.click() - # for i in range(1, 7): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.RelativeLayout[{}]'.format(i)))) - # ele.click() - # - # swipe_up(random.randint(600, 800)) - # - # # 浏览文章 - # for j in range(1, 5): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.LinearLayout[{}]'.format(j)))) - # ele.click() - # - # # 点击更多 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - # ele.click() - # # 收藏文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - # if ele.text == '收藏': - # ele.click() - # else: - # driver.tap([(18, 1235)]) - # swipe_up(800) - # # 点赞文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - # selected = ele.get_attribute("selected") - # if selected == 'false': - # ele.click() - # swipe_up(800) - # # 评论文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - # ele.click() - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - # ele.click() - # ele.send_keys(random.choice(comment_data)) - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - # ele.click() - # # 后退下滑 - # time.sleep(random.randint(2, 3)) - # driver.back() - # time.sleep(random.randint(2, 3)) - # - # # 返回 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - # ele.click() - # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + # def get_screen_size(): + # """ 屏幕大小 """ + # + # x = driver.get_window_size()['width'] + # y = driver.get_window_size()['height'] + # return x, y + # + # + # def swipe_up(t): + # """ 向上滑动，屏幕下滑 """ + # + # screen_size = get_screen_size() + # x1 = int(screen_size[0] * 0.5) # x坐标 + # y1 = int(screen_size[1] * 0.6) # 起始y坐标 + # y2 = int(screen_size[1] * 0.25) # 终点y坐标 + # driver.swipe(x1, y1, x1, y2, t) + # + # + # """ 关注 """ + # start = time.time() + # driver.back() + # # 关注列表 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + # ele.click() + # for i in range(1, 7): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.RelativeLayout[{}]'.format(i)))) + # ele.click() + # + # swipe_up(random.randint(600, 800)) + # + # # 浏览文章 + # for j in range(1, 5): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.LinearLayout[{}]'.format(j)))) + # ele.click() + # + # # 点击更多 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # ele.click() + # # 收藏文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # if ele.text == '收藏': + # ele.click() + # else: + # driver.tap([(18, 1235)]) + # swipe_up(800) + # # 点赞文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + # selected = ele.get_attribute("selected") + # if selected == 'false': + # ele.click() + # swipe_up(800) + # # 评论文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + # ele.click() + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + # ele.click() + # ele.send_keys(random.choice(comment_data)) + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + # ele.click() + # # 后退下滑 + # time.sleep(random.randint(2, 3)) + # driver.back() + # time.sleep(random.randint(2, 3)) + # + # # 返回 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) + # ele.click() + # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) - """ 退出APP """ - driver.close_app() - driver.quit() + """ 退出APP """ + driver.close_app() + driver.quit() + except: + pass From 9bdc62c697ef5ab1ab4be3b4bbd57afcd126b4f4 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 24 Jul 2019 10:37:12 +0800 Subject: [PATCH 123/148] refactor jianshu --- spider/jianshu/aium.py | 1 + 1 file changed, 1 insertion(+) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 7a773f6..2297adb 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -629,4 +629,5 @@ driver.close_app() driver.quit() except: + print("Role {} 操作异常".format(role)) pass From 5cbe82eb91741e5c3884fc3d1255d782215ed497 Mon Sep 17 00:00:00 2001 From: Jared Date: Wed, 24 Jul 2019 11:59:56 +0800 Subject: [PATCH 124/148] refactor jianshu --- spider/jianshu/aium.py | 4 ++-- spider/jianshu/jsloader.py | 18 ++++++++++++------ spider/jianshu/sium.py | 5 +++-- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 2297adb..04d82dd 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -22,9 +22,9 @@ data = jsloader.load_account("jsdata") comment_data = jsloader.load_comment() -for row in data: +for account in data: start = time.time() - username, password, role, post_num = row + username, password, role, post_num = account.username, account.password, account.role, account.post_num post_num = int(post_num) if post_num <= 0: break diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index a9627f8..4ffa76a 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -5,14 +5,22 @@ base = "D:\\Data\\" +class Account: + + def __init__(self, username, password, role, post_num, like=None) -> None: + self.username, self.password, self.role, self.post_num, self.like = username, password, role, post_num, like + + def load_account(file_name): """ 加载账号 """ print("开始读取") - data = csv.reader(open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8')) - # for row in data: - # username, password, role = row - # print("Username: {} Password: {} Role: {}".format(username, password, role)) + data = list() + reader_data = csv.reader(open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8')) + for row in reader_data: + username, password, role, post_num = row + print("Username: {} Password: {} Role: {}".format(username, password, role)) + data.append(Account(username=username, password=password, role=role, post_num=post_num)) print("读取结束") return data @@ -46,5 +54,3 @@ def load_comment(): shuffle(data) print("读取结束") return data - - diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index 2a5852f..a88b725 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -64,8 +64,9 @@ def sleep(min_seconds=3, max_seconds=10): data = jsloader.load_account("dbdata") -for row in data: - username, password, role, post_num, like = row +for account in data: + username, password, role, post_num, like = account.username, account.password, \ + account.role, account.post_num, account.like post_num = int(post_num) if post_num <= 0 and like != "T": continue From 3124eb71cd54adb4b2b9c34c7b91be1bb5fedfe7 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 25 Jul 2019 11:21:21 +0800 Subject: [PATCH 125/148] refactor jianshu --- spider/jianshu/aium.py | 1042 ++++++++++++++++++------------------ spider/jianshu/jsloader.py | 21 +- 2 files changed, 534 insertions(+), 529 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 04d82dd..ab8c221 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -22,132 +22,61 @@ data = jsloader.load_account("jsdata") comment_data = jsloader.load_comment() -for account in data: - start = time.time() - username, password, role, post_num = account.username, account.password, account.role, account.post_num - post_num = int(post_num) - if post_num <= 0: - break - print("\nNumber{}.".format(role)) - - try: - driver = webdriver.Remote(server, desired_capabilities) - wait = WebDriverWait(driver, 30) - short_wait = WebDriverWait(driver, 10) - long_wait = WebDriverWait(driver, 60) - max_long_wait = WebDriverWait(driver, 100) - - # 我的 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) - ele.click() - - # 登录 - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) - ele.send_keys(username) - ele = wait.until( - EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) - ele.send_keys(password) - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) - ele.click() - print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) +while len(data) > 0: + for account in data: + start = time.time() + username, password, role, post_num = account.username, account.password, account.role, account.post_num + if post_num <= 0: + data.remove(account) + break + print("\nNumber{}.".format(role)) try: - # 领钻弹框 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) - except: - pass - else: + driver = webdriver.Remote(server, desired_capabilities) + wait = WebDriverWait(driver, 30) + short_wait = WebDriverWait(driver, 10) + long_wait = WebDriverWait(driver, 60) + max_long_wait = WebDriverWait(driver, 100) + + # 我的 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) ele.click() - # 简书钻 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) - ele.click() - # 持有借钻 - time.sleep(5) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[15]'))) - hole_jz_num = 0 - if ele.text.find("持有借钻") != -1: - hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - print(ele.text, "/", hole_jz_num) - try: - """ 签到 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[1]'))) + # 登录 + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) ele.click() - # 每日签到 - ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[1]'))) - except: - pass - else: + ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) + ele.send_keys(username) + ele = wait.until( + EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) + ele.send_keys(password) + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() - finally: - print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) - """ 领钻 """ - start = time.time() + try: + # 领钻弹框 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) + except: + pass + else: + ele.click() + + # 简书钻 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) + ele.click() + # 持有借钻 + time.sleep(5) ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' 'android.widget.FrameLayout/' 'android.widget.RelativeLayout/' 'android.view.ViewGroup/' @@ -156,55 +85,62 @@ 'android.webkit.WebView/' 'android.view.View[1]/' 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.view.View[1]'))) - receive = 0 - jz_num = int(ele.text) - while hole_jz_num < 5000 and jz_num > 0 and receive < 10: - # 领取 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.widget.Button[1]'))) + 'android.view.View[15]'))) + hole_jz_num = 10000 + if ele.text.find("持有借钻") != -1: + hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) + print(ele.text, "/", hole_jz_num) + try: + """ 签到 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[1]'))) ele.click() - # 完成 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[2]/' - 'android.widget.Button'))) + # 每日签到 + ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[1]'))) + except: + pass + else: ele.click() + finally: + print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + + """ 领钻 """ + start = time.time() ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -224,176 +160,204 @@ 'android.view.View[2]/' 'android.view.View[5]/' 'android.view.View[1]'))) + receive = 0 jz_num = int(ele.text) - receive += 1 - print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) + while hole_jz_num < 5000 and jz_num > 0 and receive < 10: + # 领取 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.widget.Button[1]'))) + ele.click() + # 完成 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.view.View[1]'))) + jz_num = int(ele.text) + receive += 1 + print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) - # 后退 - time.sleep(2) - driver.back() + # 后退 + time.sleep(2) + driver.back() - """ 贝转钻 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[10]'))) - jsb_num = floor(float(ele.text)) - if jsb_num > 0: - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[12]'))) - ele.click() - # 输入数量 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.widget.EditText'))) - ele.click() - ele.send_keys(jsb_num) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.widget.Button'))) - ele.click() - # 确认转换 - print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) + """ 贝转钻 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[10]'))) + jsb_num = floor(float(ele.text)) + if jsb_num > 0: + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[12]'))) + ele.click() + # 输入数量 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.widget.EditText'))) + ele.click() + ele.send_keys(jsb_num) + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.widget.Button'))) + ele.click() + # 确认转换 + print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) - """ 抽奖 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - while True: - # 1. GO - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + """ 抽奖 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() - - keep_watch = False - try: - # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) - ele = short_wait.until( - EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]'))) - except: - pass - else: - text = ele.text - print(text) - if text.find('今日已用完') != -1: - break + while True: + # 1. GO + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() - keep_watch = True - try: - # 3. 关闭广告 - ele = max_long_wait.until( - EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - if not keep_watch: - # 4. 知道了 + keep_watch = False + try: + # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) ele = short_wait.until( EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -409,76 +373,82 @@ 'android.widget.FrameLayout/' 'android.webkit.WebView/' 'android.webkit.WebView/' - 'android.view.View[1]/' + 'android.view.View/' 'android.view.View[1]/' 'android.view.View[2]/' 'android.view.View[2]/' - 'android.widget.Button'))) + 'android.view.View[2]'))) + except: + pass + else: + text = ele.text + print(text) + if text.find('今日已用完') != -1: + break + ele.click() + keep_watch = True + + try: + # 3. 关闭广告 + ele = max_long_wait.until( + EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + if not keep_watch: + # 4. 知道了 + ele = short_wait.until( + EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() + except: + print("奖励异常，后退重进") + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + ele = long_wait.until( + EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() - except: - print("奖励异常，后退重进") - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - ele = long_wait.until( - EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) - """ 发文 """ - start = time.time() - driver.back() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - # 我的文章 - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.LinearLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[2]/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout/' - 'android.widget.RelativeLayout[1]/' - 'android.widget.RelativeLayout'))) - ele.click() - # 私密文章 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' - 'android.widget.LinearLayout/' - 'android.support.v7.app.ActionBar.Tab[2]'))) - ele.click() - # 循环发布 - for i in range(0, post_num): - # 进入文章 + """ 发文 """ + start = time.time() + driver.back() + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) + # 我的文章 + ele.click() ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -486,148 +456,180 @@ 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.view.ViewGroup/' 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[1]'))) - ele.click() - # 发布 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) + 'android.widget.LinearLayout[2]/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout/' + 'android.widget.RelativeLayout[1]/' + 'android.widget.RelativeLayout'))) ele.click() - # 点赞 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.ScrollView/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[3]/' - 'android.widget.RelativeLayout/' - 'android.widget.FrameLayout/android.widget.ImageView'))) + # 私密文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.FrameLayout/' + 'android.widget.FrameLayout/android.widget.FrameLayout/' + 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' + 'android.widget.LinearLayout/' + 'android.support.v7.app.ActionBar.Tab[2]'))) ele.click() - try: - # 返回 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + # 循环发布 + for i in range(0, post_num): + # 进入文章 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, + '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.view.ViewGroup/android.widget.FrameLayout/' + 'android.support.v7.widget.RecyclerView/' + 'android.widget.RelativeLayout[1]'))) ele.click() - # 点击更多 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # 发布 + ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) ele.click() - # 收藏文章 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # 点赞 + ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.ScrollView/' + 'android.widget.LinearLayout/' + 'android.widget.LinearLayout[1]/' + 'android.widget.FrameLayout[3]/' + 'android.widget.RelativeLayout/' + 'android.widget.FrameLayout/android.widget.ImageView'))) ele.click() - except: + try: + # 返回 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + ele.click() + # 点击更多 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + ele.click() + except: + driver.back() + # 返回 + time.sleep(2) driver.back() - # 返回 - time.sleep(2) - driver.back() - print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) + account.post_num = account.post_num - 1 + print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - # def get_screen_size(): - # """ 屏幕大小 """ - # - # x = driver.get_window_size()['width'] - # y = driver.get_window_size()['height'] - # return x, y - # - # - # def swipe_up(t): - # """ 向上滑动，屏幕下滑 """ - # - # screen_size = get_screen_size() - # x1 = int(screen_size[0] * 0.5) # x坐标 - # y1 = int(screen_size[1] * 0.6) # 起始y坐标 - # y2 = int(screen_size[1] * 0.25) # 终点y坐标 - # driver.swipe(x1, y1, x1, y2, t) - # - # - # """ 关注 """ - # start = time.time() - # driver.back() - # # 关注列表 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - # ele.click() - # for i in range(1, 7): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.RelativeLayout[{}]'.format(i)))) - # ele.click() - # - # swipe_up(random.randint(600, 800)) - # - # # 浏览文章 - # for j in range(1, 5): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.LinearLayout[{}]'.format(j)))) - # ele.click() - # - # # 点击更多 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - # ele.click() - # # 收藏文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - # if ele.text == '收藏': - # ele.click() - # else: - # driver.tap([(18, 1235)]) - # swipe_up(800) - # # 点赞文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - # selected = ele.get_attribute("selected") - # if selected == 'false': - # ele.click() - # swipe_up(800) - # # 评论文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - # ele.click() - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - # ele.click() - # ele.send_keys(random.choice(comment_data)) - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - # ele.click() - # # 后退下滑 - # time.sleep(random.randint(2, 3)) - # driver.back() - # time.sleep(random.randint(2, 3)) - # - # # 返回 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - # ele.click() - # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + # def get_screen_size(): + # """ 屏幕大小 """ + # + # x = driver.get_window_size()['width'] + # y = driver.get_window_size()['height'] + # return x, y + # + # + # def swipe_up(t): + # """ 向上滑动，屏幕下滑 """ + # + # screen_size = get_screen_size() + # x1 = int(screen_size[0] * 0.5) # x坐标 + # y1 = int(screen_size[1] * 0.6) # 起始y坐标 + # y2 = int(screen_size[1] * 0.25) # 终点y坐标 + # driver.swipe(x1, y1, x1, y2, t) + # + # + # """ 关注 """ + # start = time.time() + # driver.back() + # # 关注列表 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) + # ele.click() + # for i in range(1, 7): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.RelativeLayout[{}]'.format(i)))) + # ele.click() + # + # swipe_up(random.randint(600, 800)) + # + # # 浏览文章 + # for j in range(1, 5): + # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v4.view.ViewPager/' + # 'android.widget.FrameLayout/' + # 'android.widget.FrameLayout/' + # 'android.widget.LinearLayout/' + # 'android.view.ViewGroup/' + # 'android.support.v7.widget.RecyclerView/' + # 'android.widget.LinearLayout[{}]'.format(j)))) + # ele.click() + # + # # 点击更多 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # ele.click() + # # 收藏文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # if ele.text == '收藏': + # ele.click() + # else: + # driver.tap([(18, 1235)]) + # swipe_up(800) + # # 点赞文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) + # selected = ele.get_attribute("selected") + # if selected == 'false': + # ele.click() + # swipe_up(800) + # # 评论文章 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) + # ele.click() + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) + # ele.click() + # ele.send_keys(random.choice(comment_data)) + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) + # ele.click() + # # 后退下滑 + # time.sleep(random.randint(2, 3)) + # driver.back() + # time.sleep(random.randint(2, 3)) + # + # # 返回 + # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) + # ele.click() + # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) - """ 退出APP """ - driver.close_app() - driver.quit() - except: - print("Role {} 操作异常".format(role)) - pass + """ 退出APP """ + driver.close_app() + driver.quit() + except Exception as e: + print("Role {} 操作异常".format(role), e) + pass diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index 4ffa76a..485357f 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -8,7 +8,8 @@ class Account: def __init__(self, username, password, role, post_num, like=None) -> None: - self.username, self.password, self.role, self.post_num, self.like = username, password, role, post_num, like + self.username, self.password = username, password + self.role, self.post_num, self.like = int(role), int(post_num), like def load_account(file_name): @@ -16,11 +17,12 @@ def load_account(file_name): print("开始读取") data = list() - reader_data = csv.reader(open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8')) - for row in reader_data: - username, password, role, post_num = row - print("Username: {} Password: {} Role: {}".format(username, password, role)) - data.append(Account(username=username, password=password, role=role, post_num=post_num)) + with open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8') as file: + reader_data = csv.reader(file) + for row in reader_data: + username, password, role, post_num = row + print("Username: {} Password: {} Role: {}".format(username, password, role)) + data.append(Account(username=username, password=password, role=role, post_num=post_num)) print("读取结束") return data @@ -48,9 +50,10 @@ def load_comment(): print("开始读取") data = list() - reader = csv.reader(open('{}{}.csv'.format(base, "comment"), "r", encoding='utf-8')) - for row in reader: - data.append(row[0]) + with open('{}{}.csv'.format(base, "comment"), "r", encoding='utf-8') as file: + reader_data = csv.reader(file) + for row in reader_data: + data.append(row[0]) shuffle(data) print("读取结束") return data From 8b10c4614d8dffa674d3dc8c07e09874b37b94a9 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 25 Jul 2019 16:42:30 +0800 Subject: [PATCH 126/148] refactor jianshu --- spider/jianshu/aium.py | 53 ++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index ab8c221..1583479 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -1,4 +1,4 @@ -import random +import logging import time from appium import webdriver @@ -9,6 +9,7 @@ import jsloader +logging.basicConfig(level='INFO', filename='aium.log', format='%(asctime)s[%(lineno)d]: %(message)s') server = 'http://localhost:4723/wd/hub' desired_capabilities = { "platformName": "Android", @@ -29,7 +30,7 @@ if post_num <= 0: data.remove(account) break - print("\nNumber{}.".format(role)) + logging.info("\nNumber{}.".format(role)) try: driver = webdriver.Remote(server, desired_capabilities) @@ -54,7 +55,7 @@ ele.send_keys(password) ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) ele.click() - print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.登录用时{}秒".format(role, int(time.time() - start))) try: # 领钻弹框 @@ -67,6 +68,26 @@ # 简书钻 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() + # 当天收益 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[4]/' + 'android.view.View[1]/android.view.View'))) + logging.info(ele.text) # 持有借钻 time.sleep(5) ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' @@ -89,7 +110,7 @@ hole_jz_num = 10000 if ele.text.find("持有借钻") != -1: hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - print(ele.text, "/", hole_jz_num) + logging.info(ele.text, "/", hole_jz_num) try: """ 签到 """ start = time.time() @@ -137,7 +158,7 @@ else: ele.click() finally: - print("Number{}.签到用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.签到用时{}秒".format(role, int(time.time() - start))) """ 领钻 """ start = time.time() @@ -227,7 +248,7 @@ 'android.view.View[1]'))) jz_num = int(ele.text) receive += 1 - print("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) # 后退 time.sleep(2) @@ -312,7 +333,7 @@ 'android.widget.Button'))) ele.click() # 确认转换 - print("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) """ 抽奖 """ start = time.time() @@ -382,7 +403,7 @@ pass else: text = ele.text - print(text) + logging.info(text) if text.find('今日已用完') != -1: break ele.click() @@ -417,7 +438,7 @@ 'android.widget.Button'))) ele.click() except: - print("奖励异常，后退重进") + logging.info("奖励异常，后退重进") ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) ele.click() ele = long_wait.until( @@ -441,7 +462,7 @@ 'android.view.View[3]'))) ele.click() - print("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) """ 发文 """ start = time.time() @@ -473,7 +494,8 @@ 'android.widget.FrameLayout/android.widget.FrameLayout/' 'android.widget.LinearLayout/android.widget.FrameLayout/' 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.HorizontalScrollView/' + 'android.widget.LinearLayout/' + 'android.widget.HorizontalScrollView/' 'android.widget.LinearLayout/' 'android.support.v7.app.ActionBar.Tab[2]'))) ele.click() @@ -512,7 +534,8 @@ 'android.widget.LinearLayout[1]/' 'android.widget.FrameLayout[3]/' 'android.widget.RelativeLayout/' - 'android.widget.FrameLayout/android.widget.ImageView'))) + 'android.widget.FrameLayout/' + 'android.widget.ImageView'))) ele.click() try: # 返回 @@ -530,7 +553,7 @@ time.sleep(2) driver.back() account.post_num = account.post_num - 1 - print("Number{}.发文用时{}秒".format(role, int(time.time() - start))) + logging.info("Number{}.发文用时{}秒".format(role, int(time.time() - start))) # def get_screen_size(): # """ 屏幕大小 """ @@ -625,11 +648,11 @@ # # 返回 # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) # ele.click() - # print("Number{}.关注用时{}秒".format(role, int(time.time() - start))) + # logging.info("Number{}.关注用时{}秒".format(role, int(time.time() - start))) """ 退出APP """ driver.close_app() driver.quit() except Exception as e: - print("Role {} 操作异常".format(role), e) + logging.info("Role {} 操作异常".format(role), e) pass From 2cae297f9a14cce138bc878809cc945d3f19c827 Mon Sep 17 00:00:00 2001 From: Jared Date: Thu, 25 Jul 2019 16:46:00 +0800 Subject: [PATCH 127/148] refactor jianshu --- spider/jianshu/aium.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 1583479..14bb456 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -539,8 +539,10 @@ ele.click() try: # 返回 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) - ele.click() + time.sleep(2) + driver.back() + # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) + # ele.click() # 点击更多 ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) ele.click() @@ -548,7 +550,7 @@ ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) ele.click() except: - driver.back() + pass # 返回 time.sleep(2) driver.back() @@ -655,4 +657,3 @@ driver.quit() except Exception as e: logging.info("Role {} 操作异常".format(role), e) - pass From 0e89a1dc6c988e2f5ff91ddd533c37c2244bef4b Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 26 Jul 2019 10:27:28 +0800 Subject: [PATCH 128/148] refactor jianshu --- spider/jianshu/aium.py | 16 ++++++++++++---- spider/jianshu/jsloader.py | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 14bb456..487dee7 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -9,7 +9,15 @@ import jsloader -logging.basicConfig(level='INFO', filename='aium.log', format='%(asctime)s[%(lineno)d]: %(message)s') +logger = logging.getLogger() +logger.setLevel('INFO') +formatter = logging.Formatter("%(asctime)s[%(lineno)d]: %(message)s", '%Y-%m-%d %H:%M:%S') +stream_handler = logging.StreamHandler() # 输出到控制台的handler +stream_handler.setFormatter(formatter) +file_handler = logging.FileHandler('aium.log') # 输出到文件的handler +file_handler.setFormatter(formatter) +logger.addHandler(stream_handler) +logger.addHandler(file_handler) server = 'http://localhost:4723/wd/hub' desired_capabilities = { "platformName": "Android", @@ -30,7 +38,7 @@ if post_num <= 0: data.remove(account) break - logging.info("\nNumber{}.".format(role)) + logging.info("Number{}.".format(role)) try: driver = webdriver.Remote(server, desired_capabilities) @@ -110,7 +118,7 @@ hole_jz_num = 10000 if ele.text.find("持有借钻") != -1: hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - logging.info(ele.text, "/", hole_jz_num) + logging.info(ele.text + "/" + str(hole_jz_num)) try: """ 签到 """ start = time.time() @@ -656,4 +664,4 @@ driver.close_app() driver.quit() except Exception as e: - logging.info("Role {} 操作异常".format(role), e) + logging.error("Role {} 操作异常".format(str(role)), e) diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index 485357f..4c9d3d3 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -10,6 +10,7 @@ class Account: def __init__(self, username, password, role, post_num, like=None) -> None: self.username, self.password = username, password self.role, self.post_num, self.like = int(role), int(post_num), like + print("Username: {} Password: {} Role: {} Post: {} Like: {}".format(username, password, role, post_num, like)) def load_account(file_name): @@ -21,7 +22,6 @@ def load_account(file_name): reader_data = csv.reader(file) for row in reader_data: username, password, role, post_num = row - print("Username: {} Password: {} Role: {}".format(username, password, role)) data.append(Account(username=username, password=password, role=role, post_num=post_num)) print("读取结束") return data From 2f21dd6467a15f43607c8efeb51fce2dbcaf80f6 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 26 Jul 2019 13:34:40 +0800 Subject: [PATCH 129/148] refactor jianshu --- spider/jianshu/aium.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 487dee7..d48164c 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -40,8 +40,8 @@ break logging.info("Number{}.".format(role)) + driver = webdriver.Remote(server, desired_capabilities) try: - driver = webdriver.Remote(server, desired_capabilities) wait = WebDriverWait(driver, 30) short_wait = WebDriverWait(driver, 10) long_wait = WebDriverWait(driver, 60) @@ -261,6 +261,7 @@ # 后退 time.sleep(2) driver.back() + time.sleep(2) """ 贝转钻 """ start = time.time() @@ -344,6 +345,7 @@ logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) """ 抽奖 """ + time.sleep(3) start = time.time() ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -547,20 +549,20 @@ ele.click() try: # 返回 - time.sleep(2) + time.sleep(3) driver.back() # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) # ele.click() # 点击更多 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - ele.click() + # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + # ele.click() + # # 收藏文章 + # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) + # ele.click() except: pass # 返回 - time.sleep(2) + time.sleep(3) driver.back() account.post_num = account.post_num - 1 logging.info("Number{}.发文用时{}秒".format(role, int(time.time() - start))) @@ -659,9 +661,9 @@ # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) # ele.click() # logging.info("Number{}.关注用时{}秒".format(role, int(time.time() - start))) - + except Exception as e: + logging.error("Role {} 操作异常".format(str(role)), e) + finally: """ 退出APP """ driver.close_app() driver.quit() - except Exception as e: - logging.error("Role {} 操作异常".format(str(role)), e) From 03117fe73406967b4a5bce41347b2b7a21a142e7 Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 26 Jul 2019 14:20:04 +0800 Subject: [PATCH 130/148] refactor jianshu --- spider/jianshu/aium.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index d48164c..769dbfe 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -8,16 +8,9 @@ from selenium.webdriver.support.ui import WebDriverWait import jsloader +from logger import log -logger = logging.getLogger() -logger.setLevel('INFO') -formatter = logging.Formatter("%(asctime)s[%(lineno)d]: %(message)s", '%Y-%m-%d %H:%M:%S') -stream_handler = logging.StreamHandler() # 输出到控制台的handler -stream_handler.setFormatter(formatter) -file_handler = logging.FileHandler('aium.log') # 输出到文件的handler -file_handler.setFormatter(formatter) -logger.addHandler(stream_handler) -logger.addHandler(file_handler) +log.Logger() server = 'http://localhost:4723/wd/hub' desired_capabilities = { "platformName": "Android", From c4d7cb499a166368df362bd2ebf396127309d78d Mon Sep 17 00:00:00 2001 From: Jared Date: Fri, 26 Jul 2019 14:21:43 +0800 Subject: [PATCH 131/148] refactor jianshu --- logger/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logger/log.py b/logger/log.py index ae9d05e..436766e 100644 --- a/logger/log.py +++ b/logger/log.py @@ -5,8 +5,8 @@ import logging.config -class Logger(object): - def __init__(self): +class Logger: + def __init__(self, log_name='info.log'): """日志相关配置""" path = 'logs' @@ -19,7 +19,7 @@ def __init__(self): config = yaml.load(f) logging.config.dictConfig(config) else: - logging.basicConfig(level='INFO', filename='info.log', + logging.basicConfig(level='INFO', filename=log_name, format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') logging.info(">>> Load logger config") From fe3484b638649664475d16717e155fb4463b82ee Mon Sep 17 00:00:00 2001 From: Jared Date: Sat, 27 Jul 2019 14:15:49 +0800 Subject: [PATCH 132/148] update jianshu --- logger/log.py | 2 +- spider/jianshu/aium.py | 56 ++++++++++++++++++++++-------------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/logger/log.py b/logger/log.py index 436766e..4408821 100644 --- a/logger/log.py +++ b/logger/log.py @@ -13,7 +13,7 @@ def __init__(self, log_name='info.log'): if not os.path.exists(path): os.mkdir(path) - path = 'logging.yml' + path = os.path.dirname(__file__) + '/logging.yml' if os.path.exists(path): with open(path, 'r', encoding='utf-8') as f: config = yaml.load(f) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 769dbfe..67b51a4 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -338,26 +338,26 @@ logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) """ 抽奖 """ - time.sleep(3) + time.sleep(5) start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) ele.click() while True: # 1. GO @@ -444,6 +444,7 @@ logging.info("奖励异常，后退重进") ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) ele.click() + time.sleep(3) ele = long_wait.until( EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -535,27 +536,28 @@ 'android.widget.ScrollView/' 'android.widget.LinearLayout/' 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[3]/' + 'android.widget.FrameLayout[2]/' 'android.widget.RelativeLayout/' 'android.widget.FrameLayout/' 'android.widget.ImageView'))) ele.click() try: # 返回 - time.sleep(3) + time.sleep(2) driver.back() + time.sleep(2) # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) # ele.click() # 点击更多 - # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - # ele.click() - # # 收藏文章 - # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - # ele.click() + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) + ele.click() + # 收藏文章 + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/menu_bookmark'))) + ele.click() except: pass # 返回 - time.sleep(3) + time.sleep(2) driver.back() account.post_num = account.post_num - 1 logging.info("Number{}.发文用时{}秒".format(role, int(time.time() - start))) From c34d87ea44a42525a19db6a3d502e5eaa171a1f9 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 28 Jul 2019 17:10:42 +0800 Subject: [PATCH 133/148] update jianshu --- spider/jianshu/aium.py | 511 +++++++++++++++++++------------------ spider/jianshu/jsloader.py | 8 +- 2 files changed, 264 insertions(+), 255 deletions(-) diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 67b51a4..2eb669d 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -69,7 +69,8 @@ # 简书钻 ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) ele.click() - # 当天收益 + + """ 当天收益 """ ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -89,7 +90,8 @@ 'android.view.View[4]/' 'android.view.View[1]/android.view.View'))) logging.info(ele.text) - # 持有借钻 + + """ 持有借钻 """ time.sleep(5) ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -112,6 +114,7 @@ if ele.text.find("持有借钻") != -1: hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) logging.info(ele.text + "/" + str(hole_jz_num)) + try: """ 签到 """ start = time.time() @@ -161,8 +164,73 @@ finally: logging.info("Number{}.签到用时{}秒".format(role, int(time.time() - start))) - """ 领钻 """ - start = time.time() + """ 领钻 """ + start = time.time() + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.view.View[1]'))) + receive = 0 + jz_num = int(ele.text) + while hole_jz_num < 5000 and jz_num > 0 and receive < 10: + # 领取 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[5]/' + 'android.widget.Button[1]'))) + ele.click() + # 完成 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[3]/' + 'android.view.View/' + 'android.view.View[2]/' + 'android.widget.Button'))) + ele.click() ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -182,206 +250,96 @@ 'android.view.View[2]/' 'android.view.View[5]/' 'android.view.View[1]'))) - receive = 0 jz_num = int(ele.text) - while hole_jz_num < 5000 and jz_num > 0 and receive < 10: - # 领取 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.widget.Button[1]'))) - ele.click() - # 完成 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[2]/' - 'android.widget.Button'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.view.View[1]'))) - jz_num = int(ele.text) - receive += 1 - logging.info("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) - - # 后退 - time.sleep(2) - driver.back() - time.sleep(2) + receive += 1 + logging.info("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) - """ 贝转钻 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[10]'))) - jsb_num = floor(float(ele.text)) - if jsb_num > 0: - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[12]'))) - ele.click() - # 输入数量 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.widget.EditText'))) - ele.click() - ele.send_keys(jsb_num) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.widget.Button'))) - ele.click() - # 确认转换 - logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) + # 后退 + time.sleep(2) + driver.back() + time.sleep(5) - """ 抽奖 """ - time.sleep(5) - start = time.time() + """ 抽奖 """ + start = time.time() + ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() + while True: + # 1. GO ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.LinearLayout/' 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' 'android.widget.FrameLayout/' 'android.widget.RelativeLayout/' 'android.view.ViewGroup/' 'android.widget.FrameLayout/' 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) + 'android.webkit.WebView/android.view.View/' + 'android.view.View[1]/android.view.View[1]/' + 'android.view.View[2]/android.view.View[2]'))) ele.click() - while True: - # 1. GO - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) + + keep_watch = False + try: + # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) + ele = short_wait.until( + EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.view.View[2]/' + 'android.view.View[2]'))) + except: + pass + else: + text = ele.text + logging.info(text) + if text.find('今日已用完') != -1: + break ele.click() + keep_watch = True - keep_watch = False - try: - # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) + try: + # 3. 关闭广告 + ele = max_long_wait.until( + EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) + ele.click() + if not keep_watch: + # 4. 知道了 ele = short_wait.until( EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' 'android.widget.FrameLayout/' @@ -397,80 +355,127 @@ 'android.widget.FrameLayout/' 'android.webkit.WebView/' 'android.webkit.WebView/' - 'android.view.View/' + 'android.view.View[1]/' 'android.view.View[1]/' 'android.view.View[2]/' 'android.view.View[2]/' - 'android.view.View[2]'))) - except: - pass - else: - text = ele.text - logging.info(text) - if text.find('今日已用完') != -1: - break + 'android.widget.Button'))) ele.click() - keep_watch = True + except: + logging.info("奖励异常，后退重进") + ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) + ele.click() + time.sleep(3) + ele = long_wait.until( + EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[17]/' + 'android.view.View[3]'))) + ele.click() + logging.info("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) - try: - # 3. 关闭广告 - ele = max_long_wait.until( - EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - if not keep_watch: - # 4. 知道了 - ele = short_wait.until( - EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.widget.Button'))) - ele.click() - except: - logging.info("奖励异常，后退重进") - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - time.sleep(3) - ele = long_wait.until( - EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() + time.sleep(2) + driver.back() + + """ 贝转钻 """ + start = time.time() + ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[10]'))) + jsb_num = floor(float(ele.text)) + if jsb_num > 0: + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.support.v4.view.ViewPager/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[12]'))) + ele.click() + # 输入数量 + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.view.View[2]/' + 'android.widget.EditText'))) + ele.click() + ele.send_keys(jsb_num) + ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.LinearLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.FrameLayout/' + 'android.widget.RelativeLayout/' + 'android.view.ViewGroup/' + 'android.widget.FrameLayout/' + 'android.webkit.WebView/' + 'android.webkit.WebView/' + 'android.view.View[1]/' + 'android.view.View[1]/' + 'android.widget.Button'))) + ele.click() + # 确认转换 + logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) - logging.info("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) + time.sleep(2) """ 发文 """ start = time.time() - driver.back() ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) # 我的文章 ele.click() @@ -536,7 +541,7 @@ 'android.widget.ScrollView/' 'android.widget.LinearLayout/' 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[2]/' + 'android.widget.FrameLayout[3 ]/' 'android.widget.RelativeLayout/' 'android.widget.FrameLayout/' 'android.widget.ImageView'))) diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index 4c9d3d3..c7f61f1 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -21,8 +21,12 @@ def load_account(file_name): with open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8') as file: reader_data = csv.reader(file) for row in reader_data: - username, password, role, post_num = row - data.append(Account(username=username, password=password, role=role, post_num=post_num)) + if len(row) == 4: + username, password, role, post_num = row + data.append(Account(username=username, password=password, role=role, post_num=post_num)) + else: + username, password, role, post_num, like = row + data.append(Account(username=username, password=password, role=role, post_num=post_num, like=like)) print("读取结束") return data From a46c133ff3966a8288062400a975129a10c953df Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 28 Jul 2019 19:15:43 +0800 Subject: [PATCH 134/148] update jianshu --- spider/jianshu/sium.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index a88b725..a217c55 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -72,7 +72,7 @@ def sleep(min_seconds=3, max_seconds=10): continue start = time.time() options = Options() - # if role != "6": + # if role != 6: # options.add_argument('--headless') # options.add_argument('log-level=3') # options.add_argument('--no-sandbox') @@ -91,7 +91,7 @@ def sleep(min_seconds=3, max_seconds=10): try: # 登录 - if role != "6": + if role != 6: ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div[2]/div/div/ul/li[4]'))) ele.click() ele = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'douban'))) From 1b434324a5f98b0a6a4588efb0376ebe68b95186 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 29 Jul 2019 10:44:08 +0800 Subject: [PATCH 135/148] refactor jianshu --- logger/log.py | 2 +- spider/jianshu/aium.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/logger/log.py b/logger/log.py index 4408821..6efbd72 100644 --- a/logger/log.py +++ b/logger/log.py @@ -16,7 +16,7 @@ def __init__(self, log_name='info.log'): path = os.path.dirname(__file__) + '/logging.yml' if os.path.exists(path): with open(path, 'r', encoding='utf-8') as f: - config = yaml.load(f) + config = yaml.safe_load(f) logging.config.dictConfig(config) else: logging.basicConfig(level='INFO', filename=log_name, diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 2eb669d..089f61a 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -386,6 +386,7 @@ 'android.view.View[17]/' 'android.view.View[3]'))) ele.click() + print("抽奖ing...") logging.info("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) time.sleep(2) @@ -541,14 +542,14 @@ 'android.widget.ScrollView/' 'android.widget.LinearLayout/' 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[3 ]/' + 'android.widget.FrameLayout[3]/' 'android.widget.RelativeLayout/' 'android.widget.FrameLayout/' 'android.widget.ImageView'))) ele.click() try: # 返回 - time.sleep(2) + time.sleep(4) driver.back() time.sleep(2) # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) @@ -565,6 +566,7 @@ time.sleep(2) driver.back() account.post_num = account.post_num - 1 + print("发文ing...") logging.info("Number{}.发文用时{}秒".format(role, int(time.time() - start))) # def get_screen_size(): From 2c638ddacadb0a9d8e9395eae011b44f866d0c7f Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 15:16:18 +0800 Subject: [PATCH 136/148] refactor jianshu --- spider/jianshu/sium.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py index a217c55..3ef1e6c 100644 --- a/spider/jianshu/sium.py +++ b/spider/jianshu/sium.py @@ -80,7 +80,7 @@ def sleep(min_seconds=3, max_seconds=10): # options.add_argument('window-size=1366x728') # options.add_argument('--disable-dev-shm-usage') # options.add_argument('blink-settings=imagesEnabled=false') - driver = webdriver.Chrome(chrome_options=options) + driver = webdriver.Chrome(options=options) wait = WebDriverWait(driver, 30) short_wait = WebDriverWait(driver, 10) long_wait = WebDriverWait(driver, 60) From bfda94c3e4bd68275f1ac3f3ce4a43c24d5381ea Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 16:30:10 +0800 Subject: [PATCH 137/148] refactor jianshu --- logger/log.py | 1 + spider/jianshu/aium.py | 2 +- spider/jianshu/start.py | 27 --------------------------- 3 files changed, 2 insertions(+), 28 deletions(-) delete mode 100644 spider/jianshu/start.py diff --git a/logger/log.py b/logger/log.py index 6efbd72..3de26e7 100644 --- a/logger/log.py +++ b/logger/log.py @@ -18,6 +18,7 @@ def __init__(self, log_name='info.log'): with open(path, 'r', encoding='utf-8') as f: config = yaml.safe_load(f) logging.config.dictConfig(config) + logging.basicConfig(filename=log_name) else: logging.basicConfig(level='INFO', filename=log_name, format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py index 089f61a..fb8386f 100644 --- a/spider/jianshu/aium.py +++ b/spider/jianshu/aium.py @@ -10,7 +10,7 @@ import jsloader from logger import log -log.Logger() +log.Logger('aium.log') server = 'http://localhost:4723/wd/hub' desired_capabilities = { "platformName": "Android", diff --git a/spider/jianshu/start.py b/spider/jianshu/start.py deleted file mode 100644 index d1e4202..0000000 --- a/spider/jianshu/start.py +++ /dev/null @@ -1,27 +0,0 @@ -# -*- coding: utf-8 -*- -# @author : jared -# @date : 2019/5/16 19:33 - -import os -import sys - -from apscheduler.schedulers.background import * - - -def robot_job(hour): - scheduler = BlockingScheduler() - scheduler.add_job(robot, 'cron', hour=hour) - scheduler.start() - - -def robot(): - os.system('./start.sh') - - -# nohup python3 start.py 22 >> /dev/null 2>&1 & -if __name__ == '__main__': - argv = sys.argv - if len(argv) > 1: - robot_job(argv[1]) - else: - robot_job(6) From 344e3f32e6af8c6464996d1db2782e7ab100cc2d Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 16:55:43 +0800 Subject: [PATCH 138/148] refactor jianshu --- spider/jianshu/jsloader.py | 26 +++++++++++++ spider/jianshu/robot.py | 79 ++++++++++++-------------------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py index c7f61f1..0e104f9 100644 --- a/spider/jianshu/jsloader.py +++ b/spider/jianshu/jsloader.py @@ -2,6 +2,8 @@ import os from random import shuffle +from utils import rds + base = "D:\\Data\\" @@ -61,3 +63,27 @@ def load_comment(): shuffle(data) print("读取结束") return data + + +def load_server(): + """ 加载服务 """ + + print("开始读取") + with open('{}{}.csv'.format(base, "server"), "r", encoding='utf-8') as file: + reader_data = csv.reader(file) + for row in reader_data: + alias, ip, port, password = row + print("读取结束") + return alias, ip, port, password + + +def load_cookie(): + """ 加载cookie """ + + print("开始读取") + alias, ip, port, password = load_server() + redis = rds.Rds(host=ip, port=port, db=10, password=password).redis_cli + cookie_data = redis.hgetall("jianshu") + redis.close() + print("读取结束") + return cookie_data diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index d600f56..97c2bb2 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,22 +1,13 @@ import logging -import os -import platform import random -import sys +import getpass import time from selenium import webdriver from selenium.webdriver.chrome.options import Options -argv = sys.argv -argv_length = len(argv) - -username = argv[1] if argv_length > 1 else input("请输入登录账号\n") -password = argv[2] if argv_length > 2 else input("请输入登录密码\n") -role = int(argv[3]) if argv_length > 3 else input("请输入Role\n") - -if platform.system() == "Linux": - os.system("ps -ef | grep chrome | awk '{print $2}' | xargs kill -9") +import jsloader +from logger import log def sleep(min_seconds=3, max_seconds=10): @@ -24,47 +15,17 @@ def sleep(min_seconds=3, max_seconds=10): time.sleep(random.randint(min_seconds, max_seconds)) -logging.basicConfig(level='INFO', filename='robot.log', - format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') -# 无头模式 -options = Options() -options.add_argument('--headless') -options.add_argument('log-level=3') -options.add_argument('--no-sandbox') -options.add_argument('--disable-gpu') -options.add_argument('window-size=1366x728') -options.add_argument('--disable-dev-shm-usage') -options.add_argument('blink-settings=imagesEnabled=false') -driver = webdriver.Chrome(chrome_options=options) - -# 登录简书 +log.Logger('robot.log') jianshu = "https://www.jianshu.com" -driver.get("{}/sign_in".format(jianshu)) -driver.maximize_window() - -# 豆瓣登录 -driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click() -driver.find_element_by_class_name("douban").click() -sleep() - -# 切换窗口 -driver.close() -window_handles = driver.window_handles -driver.switch_to.window(window_handles[-1]) -driver.find_element_by_id("inp-alias").send_keys(username) -driver.find_element_by_id("inp-pwd").send_keys(password) - -try: - driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click() -except: - logging.error("{} : 登录失败".format(role)) -else: - sleep() - - page_title = driver.title - if page_title.find("简书") == -1: - logging.error("{} ：{}".format(role, page_title)) - else: +cookie_data = jsloader.load_cookie() +for role, cookie in cookie_data.items(): + # 无头模式 + chrome_dir = r"--user-data-dir=C:\Users\{}\AppData\Local\Google\Chrome\User Data".format(getpass.getuser()) + options = Options() + options.add_argument(chrome_dir) + options.add_argument('--headless') + driver = webdriver.Chrome(options=options) + try: # 1. 收益 driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) elements = driver.find_elements_by_class_name("order") @@ -95,6 +56,14 @@ def sleep(min_seconds=3, max_seconds=10): except: logging.error("Button_Element : {}".format(button_element)) pass -finally: - driver.close() - driver.quit() + except: + logging.error("{} : 异常".format(role)) + else: + # 重新获取 cookie + driver.add_cookie({"name": "remember_user_token", + "value": ""}) + pass + finally: + driver.delete_all_cookies() + driver.close() + driver.quit() From cd0206c064615c0808fee23e3898030d0999f10e Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 17:19:40 +0800 Subject: [PATCH 139/148] refactor jianshu --- spider/jianshu/robot.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 97c2bb2..5c0b0c5 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -1,10 +1,13 @@ +import getpass import logging import random -import getpass import time from selenium import webdriver from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.support.ui import WebDriverWait import jsloader from logger import log @@ -23,13 +26,18 @@ def sleep(min_seconds=3, max_seconds=10): chrome_dir = r"--user-data-dir=C:\Users\{}\AppData\Local\Google\Chrome\User Data".format(getpass.getuser()) options = Options() options.add_argument(chrome_dir) - options.add_argument('--headless') + # options.add_argument('--headless') driver = webdriver.Chrome(options=options) + wait = WebDriverWait(driver, 30) + driver.get(jianshu) + driver.add_cookie({"name": "remember_user_token", "value": cookie}) + sleep() + try: # 1. 收益 - driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2) - elements = driver.find_elements_by_class_name("order") - info = ' ; '.join(map(lambda ele: ele.text, elements)) + driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)) + element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'order'))) + info = ' ; '.join(map(lambda ele: ele.text, element)) logging.info("{} : {}".format(role, info)) sleep() @@ -51,8 +59,10 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)), sleep(3, 5) button_element = 'like-button-{}'.format(button_id) try: - driver.find_element_by_id(button_element).click(), sleep(3, 5) - driver.find_element_by_id(button_element).click() + element = wait.until(EC.element_to_be_clickable((By.ID, button_element))) + element.click() + sleep(3, 5) + element.click() except: logging.error("Button_Element : {}".format(button_element)) pass @@ -60,8 +70,7 @@ def sleep(min_seconds=3, max_seconds=10): logging.error("{} : 异常".format(role)) else: # 重新获取 cookie - driver.add_cookie({"name": "remember_user_token", - "value": ""}) + driver.add_cookie({"name": "remember_user_token", "value": ""}) pass finally: driver.delete_all_cookies() From 35208e8fe60b632fff0a739876eb9fc2d72e5a88 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 17:20:54 +0800 Subject: [PATCH 140/148] refactor jianshu --- spider/jianshu/robot.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 5c0b0c5..954736a 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -66,12 +66,9 @@ def sleep(min_seconds=3, max_seconds=10): except: logging.error("Button_Element : {}".format(button_element)) pass - except: - logging.error("{} : 异常".format(role)) - else: - # 重新获取 cookie + except Exception: + logging.error("{} 重新获取 cookie".format(role), Exception) driver.add_cookie({"name": "remember_user_token", "value": ""}) - pass finally: driver.delete_all_cookies() driver.close() From bb41ebe0a5f004c189c0a3d025548dbcfce8eb25 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 17:30:47 +0800 Subject: [PATCH 141/148] refactor log --- logger/log.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/logger/log.py b/logger/log.py index 3de26e7..4422c59 100644 --- a/logger/log.py +++ b/logger/log.py @@ -22,5 +22,3 @@ def __init__(self, log_name='info.log'): else: logging.basicConfig(level='INFO', filename=log_name, format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s') - - logging.info(">>> Load logger config") From 0491f4d1756a31f9e43099444b8693582c54485a Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 17:49:10 +0800 Subject: [PATCH 142/148] refactor log --- spider/jianshu/robot.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index 954736a..a38c412 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -30,16 +30,16 @@ def sleep(min_seconds=3, max_seconds=10): driver = webdriver.Chrome(options=options) wait = WebDriverWait(driver, 30) driver.get(jianshu) + driver.maximize_window() driver.add_cookie({"name": "remember_user_token", "value": cookie}) sleep() try: # 1. 收益 driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)) - element = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'order'))) - info = ' ; '.join(map(lambda ele: ele.text, element)) - logging.info("{} : {}".format(role, info)) - sleep() + elements = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'order'))) + info = ' ; '.join(map(lambda ele: ele.text, elements)) + logging.info("{} ; {}".format(role, info)) # 2.消息 driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) @@ -47,7 +47,6 @@ def sleep(min_seconds=3, max_seconds=10): driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) - sleep() # 3.评论 jianshu_p = "{}/p/".format(jianshu) @@ -56,16 +55,14 @@ def sleep(min_seconds=3, max_seconds=10): "9aaebad1753a": "42635258", "f3398c00ffc1": "42854890", "e369bb81bcc8": "42855033", "4ec116446717": "42855114", "2d184d128522": "42855181", } for comment_id, button_id in comments.items(): - driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)), sleep(3, 5) - button_element = 'like-button-{}'.format(button_id) + driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)) try: - element = wait.until(EC.element_to_be_clickable((By.ID, button_element))) - element.click() + like_ele = wait.until(EC.element_to_be_clickable((By.ID, 'like-button-{}'.format(button_id)))) + like_ele.click() sleep(3, 5) - element.click() - except: - logging.error("Button_Element : {}".format(button_element)) - pass + like_ele.click() + except Exception: + logging.error("Button_Element : {}".format(button_id), Exception) except Exception: logging.error("{} 重新获取 cookie".format(role), Exception) driver.add_cookie({"name": "remember_user_token", "value": ""}) From e4c3abf9b00db51bdfc58d752b00a539f0510684 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 19:38:50 +0800 Subject: [PATCH 143/148] refactor robot --- spider/jianshu/robot.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py index a38c412..4d8728a 100644 --- a/spider/jianshu/robot.py +++ b/spider/jianshu/robot.py @@ -30,7 +30,6 @@ def sleep(min_seconds=3, max_seconds=10): driver = webdriver.Chrome(options=options) wait = WebDriverWait(driver, 30) driver.get(jianshu) - driver.maximize_window() driver.add_cookie({"name": "remember_user_token", "value": cookie}) sleep() @@ -56,13 +55,9 @@ def sleep(min_seconds=3, max_seconds=10): "4ec116446717": "42855114", "2d184d128522": "42855181", } for comment_id, button_id in comments.items(): driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)) - try: - like_ele = wait.until(EC.element_to_be_clickable((By.ID, 'like-button-{}'.format(button_id)))) - like_ele.click() - sleep(3, 5) - like_ele.click() - except Exception: - logging.error("Button_Element : {}".format(button_id), Exception) + like_ele = wait.until(EC.element_to_be_clickable((By.ID, 'like-button-{}'.format(button_id)))) + driver.execute_script("arguments[0].click();", like_ele), sleep(3, 5) + driver.execute_script("arguments[0].click();", like_ele) except Exception: logging.error("{} 重新获取 cookie".format(role), Exception) driver.add_cookie({"name": "remember_user_token", "value": ""}) From 1936ae3c9ef145341498809d404056b10c8b3070 Mon Sep 17 00:00:00 2001 From: Jared Date: Tue, 30 Jul 2019 21:59:27 +0800 Subject: [PATCH 144/148] delete jianshu --- spider/jianshu/aium.py | 671 ------------------------------------- spider/jianshu/jsloader.py | 89 ----- spider/jianshu/robot.py | 67 ---- spider/jianshu/sium.py | 166 --------- 4 files changed, 993 deletions(-) delete mode 100644 spider/jianshu/aium.py delete mode 100644 spider/jianshu/jsloader.py delete mode 100644 spider/jianshu/robot.py delete mode 100644 spider/jianshu/sium.py diff --git a/spider/jianshu/aium.py b/spider/jianshu/aium.py deleted file mode 100644 index fb8386f..0000000 --- a/spider/jianshu/aium.py +++ /dev/null @@ -1,671 +0,0 @@ -import logging -import time - -from appium import webdriver -from math import floor -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait - -import jsloader -from logger import log - -log.Logger('aium.log') -server = 'http://localhost:4723/wd/hub' -desired_capabilities = { - "platformName": "Android", - "deviceName": "WAS_AL00", - "appPackage": "com.jianshu.haruki", - "appActivity": "com.baiji.jianshu.MainActivity", - 'unicodeKeyboard': True, # 是使用unicode编码方式发送字符串 - 'resetKeyboard': True, # 隐藏键盘 - # 'noReset': True # 不清空会话 -} - -data = jsloader.load_account("jsdata") -comment_data = jsloader.load_comment() -while len(data) > 0: - for account in data: - start = time.time() - username, password, role, post_num = account.username, account.password, account.role, account.post_num - if post_num <= 0: - data.remove(account) - break - logging.info("Number{}.".format(role)) - - driver = webdriver.Remote(server, desired_capabilities) - try: - wait = WebDriverWait(driver, 30) - short_wait = WebDriverWait(driver, 10) - long_wait = WebDriverWait(driver, 60) - max_long_wait = WebDriverWait(driver, 100) - - # 我的 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/user_top_info_avatar'))) - ele.click() - - # 登录 - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/tv_switch_login_mode'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_account'))) - ele.send_keys(username) - ele = wait.until( - EC.presence_of_element_located((By.ID, 'com.jianshu.haruki:id/et_verification_code_or_password'))) - ele.send_keys(password) - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_login'))) - ele.click() - logging.info("Number{}.登录用时{}秒".format(role, int(time.time() - start))) - - try: - # 领钻弹框 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_cancel'))) - except: - pass - else: - ele.click() - - # 简书钻 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_jsd'))) - ele.click() - - """ 当天收益 """ - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[4]/' - 'android.view.View[1]/android.view.View'))) - logging.info(ele.text) - - """ 持有借钻 """ - time.sleep(5) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[15]'))) - hole_jz_num = 10000 - if ele.text.find("持有借钻") != -1: - hole_jz_num = floor(float(ele.text.replace("持有借钻: ", ""))) - logging.info(ele.text + "/" + str(hole_jz_num)) - - try: - """ 签到 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[1]'))) - ele.click() - # 每日签到 - ele = short_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[1]'))) - except: - pass - else: - ele.click() - finally: - logging.info("Number{}.签到用时{}秒".format(role, int(time.time() - start))) - - """ 领钻 """ - start = time.time() - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.view.View[1]'))) - receive = 0 - jz_num = int(ele.text) - while hole_jz_num < 5000 and jz_num > 0 and receive < 10: - # 领取 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.widget.Button[1]'))) - ele.click() - # 完成 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[3]/' - 'android.view.View/' - 'android.view.View[2]/' - 'android.widget.Button'))) - ele.click() - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[5]/' - 'android.view.View[1]'))) - jz_num = int(ele.text) - receive += 1 - logging.info("Number{}.领钻用时{}秒".format(role, int(time.time() - start))) - - # 后退 - time.sleep(2) - driver.back() - time.sleep(5) - - """ 抽奖 """ - start = time.time() - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - while True: - # 1. GO - ele = long_wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/android.view.View/' - 'android.view.View[1]/android.view.View[1]/' - 'android.view.View[2]/android.view.View[2]'))) - ele.click() - - keep_watch = False - try: - # 2. 继续观看(根据元素文字来判断是否还有抽奖次数) - ele = short_wait.until( - EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.view.View[2]'))) - except: - pass - else: - text = ele.text - logging.info(text) - if text.find('今日已用完') != -1: - break - ele.click() - keep_watch = True - - try: - # 3. 关闭广告 - ele = max_long_wait.until( - EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tt_video_ad_close'))) - ele.click() - if not keep_watch: - # 4. 知道了 - ele = short_wait.until( - EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.view.View[2]/' - 'android.widget.Button'))) - ele.click() - except: - logging.info("奖励异常，后退重进") - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/close'))) - ele.click() - time.sleep(3) - ele = long_wait.until( - EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[17]/' - 'android.view.View[3]'))) - ele.click() - print("抽奖ing...") - logging.info("Number{}.抽奖用时{}秒".format(role, int(time.time() - start))) - - time.sleep(2) - driver.back() - - """ 贝转钻 """ - start = time.time() - ele = long_wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[10]'))) - jsb_num = floor(float(ele.text)) - if jsb_num > 0: - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[12]'))) - ele.click() - # 输入数量 - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.view.View[2]/' - 'android.widget.EditText'))) - ele.click() - ele.send_keys(jsb_num) - ele = wait.until(EC.presence_of_element_located((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.RelativeLayout/' - 'android.view.ViewGroup/' - 'android.widget.FrameLayout/' - 'android.webkit.WebView/' - 'android.webkit.WebView/' - 'android.view.View[1]/' - 'android.view.View[1]/' - 'android.widget.Button'))) - ele.click() - # 确认转换 - logging.info("Number{}.转钻用时{}秒".format(role, int(time.time() - start))) - - time.sleep(2) - - """ 发文 """ - start = time.time() - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tab_mine'))) - # 我的文章 - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.LinearLayout/' - 'android.view.ViewGroup/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.LinearLayout[2]/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout/' - 'android.widget.RelativeLayout[1]/' - 'android.widget.RelativeLayout'))) - ele.click() - # 私密文章 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/android.widget.FrameLayout/' - 'android.widget.FrameLayout/android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.HorizontalScrollView/' - 'android.widget.LinearLayout/' - 'android.support.v7.app.ActionBar.Tab[2]'))) - ele.click() - # 循环发布 - for i in range(0, post_num): - # 进入文章 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, - '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.support.v4.view.ViewPager/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.view.ViewGroup/android.widget.FrameLayout/' - 'android.support.v7.widget.RecyclerView/' - 'android.widget.RelativeLayout[1]'))) - ele.click() - # 发布 - ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_action_public'))) - ele.click() - # 点赞 - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.LinearLayout/' - 'android.widget.FrameLayout/' - 'android.widget.FrameLayout/' - 'android.widget.ScrollView/' - 'android.widget.LinearLayout/' - 'android.widget.LinearLayout[1]/' - 'android.widget.FrameLayout[3]/' - 'android.widget.RelativeLayout/' - 'android.widget.FrameLayout/' - 'android.widget.ImageView'))) - ele.click() - try: - # 返回 - time.sleep(4) - driver.back() - time.sleep(2) - # ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_nav'))) - # ele.click() - # 点击更多 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - ele.click() - # 收藏文章 - ele = short_wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/menu_bookmark'))) - ele.click() - except: - pass - # 返回 - time.sleep(2) - driver.back() - account.post_num = account.post_num - 1 - print("发文ing...") - logging.info("Number{}.发文用时{}秒".format(role, int(time.time() - start))) - - # def get_screen_size(): - # """ 屏幕大小 """ - # - # x = driver.get_window_size()['width'] - # y = driver.get_window_size()['height'] - # return x, y - # - # - # def swipe_up(t): - # """ 向上滑动，屏幕下滑 """ - # - # screen_size = get_screen_size() - # x1 = int(screen_size[0] * 0.5) # x坐标 - # y1 = int(screen_size[1] * 0.6) # 起始y坐标 - # y2 = int(screen_size[1] * 0.25) # 终点y坐标 - # driver.swipe(x1, y1, x1, y2, t) - # - # - # """ 关注 """ - # start = time.time() - # driver.back() - # # 关注列表 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/login_follow_text'))) - # ele.click() - # for i in range(1, 7): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.RelativeLayout[{}]'.format(i)))) - # ele.click() - # - # swipe_up(random.randint(600, 800)) - # - # # 浏览文章 - # for j in range(1, 5): - # ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/hierarchy/android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v4.view.ViewPager/' - # 'android.widget.FrameLayout/' - # 'android.widget.FrameLayout/' - # 'android.widget.LinearLayout/' - # 'android.view.ViewGroup/' - # 'android.support.v7.widget.RecyclerView/' - # 'android.widget.LinearLayout[{}]'.format(j)))) - # ele.click() - # - # # 点击更多 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/action_more'))) - # ele.click() - # # 收藏文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_bookmark'))) - # if ele.text == '收藏': - # ele.click() - # else: - # driver.tap([(18, 1235)]) - # swipe_up(800) - # # 点赞文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_vote_up'))) - # selected = ele.get_attribute("selected") - # if selected == 'false': - # ele.click() - # swipe_up(800) - # # 评论文章 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/tv_write_comment'))) - # ele.click() - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/edit_comment'))) - # ele.click() - # ele.send_keys(random.choice(comment_data)) - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/text_comment_send'))) - # ele.click() - # # 后退下滑 - # time.sleep(random.randint(2, 3)) - # driver.back() - # time.sleep(random.randint(2, 3)) - # - # # 返回 - # ele = wait.until(EC.element_to_be_clickable((By.ID, 'com.jianshu.haruki:id/iv_toolbar_back'))) - # ele.click() - # logging.info("Number{}.关注用时{}秒".format(role, int(time.time() - start))) - except Exception as e: - logging.error("Role {} 操作异常".format(str(role)), e) - finally: - """ 退出APP """ - driver.close_app() - driver.quit() diff --git a/spider/jianshu/jsloader.py b/spider/jianshu/jsloader.py deleted file mode 100644 index 0e104f9..0000000 --- a/spider/jianshu/jsloader.py +++ /dev/null @@ -1,89 +0,0 @@ -import csv -import os -from random import shuffle - -from utils import rds - -base = "D:\\Data\\" - - -class Account: - - def __init__(self, username, password, role, post_num, like=None) -> None: - self.username, self.password = username, password - self.role, self.post_num, self.like = int(role), int(post_num), like - print("Username: {} Password: {} Role: {} Post: {} Like: {}".format(username, password, role, post_num, like)) - - -def load_account(file_name): - """ 加载账号 """ - - print("开始读取") - data = list() - with open('{}{}.csv'.format(base, file_name), "r", encoding='utf-8') as file: - reader_data = csv.reader(file) - for row in reader_data: - if len(row) == 4: - username, password, role, post_num = row - data.append(Account(username=username, password=password, role=role, post_num=post_num)) - else: - username, password, role, post_num, like = row - data.append(Account(username=username, password=password, role=role, post_num=post_num, like=like)) - print("读取结束") - return data - - -def load_article(role): - """ 加载文章 """ - - print("开始读取") - path = "{}{}".format(base, role) - file_list = os.listdir(path) - print("文件夹路径【{}】下共有文章{}篇".format(path, len(file_list))) - shuffle(file_list) - data = dict() - for file_name in file_list: - article_name = file_name.replace(".md", "") - with open(path + "\\" + file_name, "r", encoding='utf-8') as file: - article_content = file.read() - data.setdefault(article_name, article_content) - print("读取结束".format(role)) - return data - - -def load_comment(): - """ 加载评论 """ - - print("开始读取") - data = list() - with open('{}{}.csv'.format(base, "comment"), "r", encoding='utf-8') as file: - reader_data = csv.reader(file) - for row in reader_data: - data.append(row[0]) - shuffle(data) - print("读取结束") - return data - - -def load_server(): - """ 加载服务 """ - - print("开始读取") - with open('{}{}.csv'.format(base, "server"), "r", encoding='utf-8') as file: - reader_data = csv.reader(file) - for row in reader_data: - alias, ip, port, password = row - print("读取结束") - return alias, ip, port, password - - -def load_cookie(): - """ 加载cookie """ - - print("开始读取") - alias, ip, port, password = load_server() - redis = rds.Rds(host=ip, port=port, db=10, password=password).redis_cli - cookie_data = redis.hgetall("jianshu") - redis.close() - print("读取结束") - return cookie_data diff --git a/spider/jianshu/robot.py b/spider/jianshu/robot.py deleted file mode 100644 index 4d8728a..0000000 --- a/spider/jianshu/robot.py +++ /dev/null @@ -1,67 +0,0 @@ -import getpass -import logging -import random -import time - -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.by import By -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait - -import jsloader -from logger import log - - -def sleep(min_seconds=3, max_seconds=10): - """随机休眠""" - time.sleep(random.randint(min_seconds, max_seconds)) - - -log.Logger('robot.log') -jianshu = "https://www.jianshu.com" -cookie_data = jsloader.load_cookie() -for role, cookie in cookie_data.items(): - # 无头模式 - chrome_dir = r"--user-data-dir=C:\Users\{}\AppData\Local\Google\Chrome\User Data".format(getpass.getuser()) - options = Options() - options.add_argument(chrome_dir) - # options.add_argument('--headless') - driver = webdriver.Chrome(options=options) - wait = WebDriverWait(driver, 30) - driver.get(jianshu) - driver.add_cookie({"name": "remember_user_token", "value": cookie}) - sleep() - - try: - # 1. 收益 - driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)) - elements = wait.until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'order'))) - info = ' ; '.join(map(lambda ele: ele.text, elements)) - logging.info("{} ; {}".format(role, info)) - - # 2.消息 - driver.get("{}{}".format(jianshu, "/notifications#/likes")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/others")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/follows")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/money")), sleep(1, 2) - driver.get("{}{}".format(jianshu, "/notifications#/comments")), sleep(1, 2) - - # 3.评论 - jianshu_p = "{}/p/".format(jianshu) - comments = {"ac02c56c0865": "38968576", "be27870bdba9": "38969657", "277f1b0a140d": "40692862", - "15eb212788fc": "40693328", "81d5ae85f38a": "40874580", "4dca23c2c75a": "40874520", - "9aaebad1753a": "42635258", "f3398c00ffc1": "42854890", "e369bb81bcc8": "42855033", - "4ec116446717": "42855114", "2d184d128522": "42855181", } - for comment_id, button_id in comments.items(): - driver.get("{}{}#comment-{}".format(jianshu_p, comment_id, button_id)) - like_ele = wait.until(EC.element_to_be_clickable((By.ID, 'like-button-{}'.format(button_id)))) - driver.execute_script("arguments[0].click();", like_ele), sleep(3, 5) - driver.execute_script("arguments[0].click();", like_ele) - except Exception: - logging.error("{} 重新获取 cookie".format(role), Exception) - driver.add_cookie({"name": "remember_user_token", "value": ""}) - finally: - driver.delete_all_cookies() - driver.close() - driver.quit() diff --git a/spider/jianshu/sium.py b/spider/jianshu/sium.py deleted file mode 100644 index 3ef1e6c..0000000 --- a/spider/jianshu/sium.py +++ /dev/null @@ -1,166 +0,0 @@ -# -*- coding: utf-8 -*- -# @author : jared -# @date : 2019/7/5 23:23 - -import random -import time -from bs4 import BeautifulSoup - -import pyperclip -import requests -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.common.by import By -from selenium.webdriver.common.keys import Keys -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.support.ui import WebDriverWait - -import jsloader - -jianshu = "https://www.jianshu.com" -articles = set() - - -def load_articles(): - """ 获取所有文章 """ - - if not articles: - start_load = time.time() - headers = {"user-agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36"} - user_ids = ["000a530f461c", "68a80b0c87a5", "ef6f9c30080a", "b34dcc8c2394", "37ec6dd9fc1c", - "04f7029dba14", "32bbb0ecfcb9"] - jianshu_u = "{}/u/".format(jianshu) - for uid in user_ids: - user = "{}{}".format(jianshu_u, uid) - html = requests.get(user, headers=headers).text - bs = BeautifulSoup(html, 'html.parser') - info = bs.find("div", class_="info") - p_list = info.find_all("p") - article_num = int(p_list[2].text) - - count = 0 - while count < article_num: - for page in range(1, 100): - html = requests.get("{}?order_by=shared_at&page={}".format(user, page), headers=headers).text - bs = BeautifulSoup(html, 'html.parser') - titles = bs.find_all('a', class_="title") - for article_title in titles: - article_url = article_title.get("href") - if article_url: - count += 1 - articles.add(article_url) - if count >= article_num: - break - - print("共发布了{}篇".format(len(articles))) - print("获取用时{}秒".format(int(time.time() - start_load))) - return articles - - -def sleep(min_seconds=3, max_seconds=10): - """ 随机休眠 """ - - time.sleep(random.randint(min_seconds, max_seconds)) - - -data = jsloader.load_account("dbdata") -for account in data: - username, password, role, post_num, like = account.username, account.password, \ - account.role, account.post_num, account.like - post_num = int(post_num) - if post_num <= 0 and like != "T": - continue - start = time.time() - options = Options() - # if role != 6: - # options.add_argument('--headless') - # options.add_argument('log-level=3') - # options.add_argument('--no-sandbox') - # options.add_argument('--disable-gpu') - # options.add_argument('window-size=1366x728') - # options.add_argument('--disable-dev-shm-usage') - # options.add_argument('blink-settings=imagesEnabled=false') - driver = webdriver.Chrome(options=options) - wait = WebDriverWait(driver, 30) - short_wait = WebDriverWait(driver, 10) - long_wait = WebDriverWait(driver, 60) - max_long_wait = WebDriverWait(driver, 100) - - driver.get("{}/sign_in".format(jianshu)) - driver.maximize_window() - - try: - # 登录 - if role != 6: - ele = wait.until(EC.element_to_be_clickable((By.XPATH, '/html/body/div[1]/div[2]/div/div/ul/li[4]'))) - ele.click() - ele = wait.until(EC.element_to_be_clickable((By.CLASS_NAME, 'douban'))) - ele.click() - - sleep(3, 5) - driver.close() - window_handles = driver.window_handles - driver.switch_to.window(window_handles[-1]) - - username_ele = wait.until(EC.element_to_be_clickable((By.ID, 'inp-alias'))) - password_ele = wait.until(EC.element_to_be_clickable((By.ID, 'inp-pwd'))) - login_ele = wait.until( - EC.element_to_be_clickable((By.XPATH, '/html/body/div/div[2]/form[2]/div[4]/input[1]'))) - else: - username_ele = wait.until(EC.element_to_be_clickable((By.ID, 'session_email_or_mobile_number'))) - password_ele = wait.until(EC.element_to_be_clickable((By.ID, 'session_password'))) - login_ele = wait.until(EC.element_to_be_clickable((By.ID, 'sign-in-form-submit-btn'))) - - username_ele.send_keys(username) - password_ele.send_keys(password) - login_ele.click() - except: - print("{} : 登录失败".format(role)) - else: - try: - # 通过该元素判断是否正常登录 - menu_ele = wait.until(EC.element_to_be_clickable((By.ID, 'menu'))) - except: - print("{} ：{} 登录失败".format(role, driver.title)) - else: - print("\nNumber{}.".format(role)) - print("Number{}.登录用时{}秒".format(role, int(time.time() - start))) - - """ 写文 """ - if post_num > 0: - start = time.time() - article_data = jsloader.load_article(role) - driver.get("{}/writer#/".format(jianshu)) - sleep() - for title, content in article_data.items(): - ele = wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "i[class='fa fa-plus-circle']"))) - ele.click() - sleep(2, 3) - - ele = wait.until(EC.presence_of_element_located((By.CLASS_NAME, "_24i7u"))) - ele.clear() - ele.send_keys(title) - sleep(2, 3) - - pyperclip.copy(content) - ele = wait.until(EC.presence_of_element_located((By.ID, "arthur-editor"))) - ele.clear() - ele.send_keys(Keys.CONTROL, 'v') - sleep(2, 3) - print("Number{}.写文用时{}秒".format(role, int(time.time() - start))) - - """ 喜欢 """ - if like == "T": - load_articles() - start = time.time() - for article in articles: - url = "{}{}".format(jianshu, article) - driver.get(url) - beautiful_soup = BeautifulSoup(driver.page_source, 'html.parser') - like_element = beautiful_soup.find("div", class_="btn like-group active") - if not like_element: - print("{}".format(url)) - print("Number{}.喜欢用时{}秒".format(role, int(time.time() - start))) - finally: - driver.close() - driver.quit() From 5f2bd8dd0cbe2a4c00e1ac48fc2e24e11e0f8a76 Mon Sep 17 00:00:00 2001 From: Jared Date: Mon, 30 Dec 2019 11:08:47 +0800 Subject: [PATCH 145/148] refactor avatar --- blogs/read.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/blogs/read.py b/blogs/read.py index 7d9b598..1df643a 100644 --- a/blogs/read.py +++ b/blogs/read.py @@ -1,5 +1,6 @@ -import math import time + +import math import numpy import requests from bs4 import BeautifulSoup @@ -54,7 +55,7 @@ def count(self): bs = BeautifulSoup(resp.text, 'html.parser') avatar = bs.find(class_='avatar') # 头像 - self.avatar = 'https:' + avatar.img['src'] + self.avatar = avatar.img['src'] nickname = bs.find(class_='name') # 昵称 self.nickname = nickname.text From 9a27203a5129a4da6c1ced1dc1106f76847fa166 Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 26 Jan 2020 18:06:30 +0800 Subject: [PATCH 146/148] delete jianshu --- wechat/wx_mps.py | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py index 90f3697..fdd4360 100644 --- a/wechat/wx_mps.py +++ b/wechat/wx_mps.py @@ -16,18 +16,18 @@ elastic = es.Es(host='localhost', port=12900, index='mp', doc='article') -# def load(): -# for i in range(24, 25): -# sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ -# "tb_mps where id = {} and show = True".format(i) -# result = postgres.fetch_all(sql) -# for r in result: -# r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ -# r[3], r[4], r[5] -# r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( -# r_pass_ticket, r_wap_sid2) -# mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) -# mps.start() +def load(): + for i in [13, 14, 25, 3]: + sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \ + "tb_mps where id = {} and show = True".format(i) + result = postgres.fetch_all(sql) + for r in result: + r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \ + r[3], r[4], r[5] + r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format( + r_pass_ticket, r_wap_sid2) + mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id) + mps.start() class WxMps(object): @@ -248,14 +248,14 @@ def _save_article_comment(): if __name__ == '__main__': - _id = 18 - biz = 'MjM5NjAxOTU4MA==' - app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' - # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 - pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' - wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' - cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) - wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) - # 开始爬取文章及评论 - wxMps.start() - # load() + # _id = 18 + # biz = 'MjM5NjAxOTU4MA==' + # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~' + # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关 + # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h' + # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO' + # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2) + # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie) + # # 开始爬取文章及评论 + # wxMps.start() + load() From 605373a47b9ece708909465f5d27fc035f6f3d5e Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 26 Jan 2020 18:08:27 +0800 Subject: [PATCH 147/148] delete jianshu --- wechat/wx_mps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wechat/wx_mps.sh b/wechat/wx_mps.sh index 317ffb9..96ed30e 100644 --- a/wechat/wx_mps.sh +++ b/wechat/wx_mps.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -nohup python3 wx_mps.py >> /dev/null 2>&1 & \ No newline at end of file +nohup python3 wx_mps.py >> ./logs/mps.log 2>&1 & \ No newline at end of file From dd97167bf1cd569417e035825ad8031f370e3b7b Mon Sep 17 00:00:00 2001 From: Jared Date: Sun, 26 Jan 2020 18:09:50 +0800 Subject: [PATCH 148/148] delete jianshu --- wechat/wx_mps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wechat/wx_mps.sh b/wechat/wx_mps.sh index 96ed30e..b802e48 100644 --- a/wechat/wx_mps.sh +++ b/wechat/wx_mps.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -nohup python3 wx_mps.py >> ./logs/mps.log 2>&1 & \ No newline at end of file +nohup python3 wx_mps.py >> mps.log 2>&1 & \ No newline at end of file