diff --git a/.gitignore b/.gitignore
index 62c8935..0b71a72 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
-.idea/
\ No newline at end of file
+.idea/
+spider/jianshu/start.sh
\ No newline at end of file
diff --git a/README.md b/README.md
index baa8cf5..8e4a3df 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,13 @@
-### 项目说明
-
-#### auto : python + selenium自动发布平台博客，包括简书、CSDN和OSChina
-#### baike: 百度百科和搜狗百科小爬虫
-#### mydjango : Django上手项目
-#### spider : 京东和内涵社区小爬虫
-#### jianshu : 简书数据统计
\ No newline at end of file
+## Introduction
+
+- `auto`: 自动化 && 抓包 Demo, selenium、appium、mitmproxy
+- `basic`: python基础 Demo
+- `blogs`: 基于web.py的简书数据统计
+- `datascience`: 数据科学,numpy、pandas、matplotlib
+- `mydjango`: Django Demo
+- `orm`: sqlalchemy && peewee Demo
+- `scrapys`: scrapy examples
+- `single`: 某小程序爬虫 && 机器人
+- `spider`: 简单爬虫examples
+- `utils`: 小小工具包
+- `wechat`: 微信机器人 && 公众号抓取
\ No newline at end of file
diff --git a/auto/keepium.py b/auto/keepium.py
new file mode 100644
index 0000000..8ee9059
--- /dev/null
+++ b/auto/keepium.py
@@ -0,0 +1,52 @@
+from appium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+server = 'http://localhost:4723/wd/hub' # Appium Server, 端口默认为4723
+desired_capabilities = {
+ 'platformName': 'Android',
+ 'deviceName': 'WAS_AL00', # 需替换成你的deviceName
+ 'appPackage': 'com.gotokeep.keep',
+ 'appActivity': 'com.gotokeep.keep.splash.SplashActivity'
+}
+
+driver = webdriver.Remote(server, desired_capabilities)
+wait = WebDriverWait(driver, 10) # 最大查找等待超时时间：10s
+
+
+def get_permission():
+ """允许APP获取的某些权限"""
+
+ try:
+ ask = wait.until(EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/do_not_ask_checkbox')))
+ ask.click()
+ allow = wait.until(
+ EC.presence_of_element_located((By.ID, 'com.android.packageinstaller:id/permission_allow_button')))
+ allow.click()
+ except:
+ pass
+
+
+# 允许两项授权
+get_permission()
+get_permission()
+
+# 点击“立即使用”
+welcome = wait.until(EC.presence_of_element_located((By.ID, 'com.gotokeep.keep:id/btn_bottom_in_video_welcome')))
+welcome.click()
+
+# 切换“密码登录”(同样可以使用第三方进行授权登录)
+driver.tap([(900, 110)])
+
+# 输入“手机号”
+phone = driver.find_element_by_accessibility_id('Phone Number In Login')
+phone.send_keys('13988888888') # 替换成实际的账号
+
+# 输入“密码”
+password = driver.find_element_by_accessibility_id('Password In Login')
+password.send_keys('123456') # 替换成实际的密码
+
+# 点击“登录”
+login = driver.find_element_by_id('com.gotokeep.keep:id/btn_action')
+login.click()
diff --git a/auto/mitmdemo.py b/auto/mitmdemo.py
new file mode 100644
index 0000000..c884b7c
--- /dev/null
+++ b/auto/mitmdemo.py
@@ -0,0 +1,21 @@
+import mitmproxy.http
+
+
+def request(flow: mitmproxy.http.HTTPFlow):
+ """
+ The full HTTP request has been read.
+ """
+
+ flow.request.headers["User-Agent"] = "Chrome/66.0.3497.100"
+
+ pretty_url = flow.request.pretty_url
+ print(pretty_url)
+
+
+def response(flow: mitmproxy.http.HTTPFlow):
+ """
+ The full HTTP response has been read.
+ """
+
+ content = flow.response.content
+ print(content)
diff --git a/jd/qmm.py b/auto/qmm.py
similarity index 98%
rename from jd/qmm.py
rename to auto/qmm.py
index 4639271..7591ca2 100644
--- a/jd/qmm.py
+++ b/auto/qmm.py
@@ -39,7 +39,7 @@ def _crawl_url(self):
for link in bs.tbody.find_all('a'):
text = link.text
if self.months:
- if not list(filter(lambda m: m in text, self.months)): continue
+ if not list(filter(lambda m: m in text[0:len(m)], self.months)): continue
if self.days:
if not list(filter(lambda d: d in text, self.days)): continue
@@ -172,5 +172,5 @@ def _finance_sign(self, driver):
if __name__ == '__main__':
- qmm = QMM(sleep=3, months='10', days='25-30')
+ qmm = QMM(sleep=3, months='4', days='1-30')
qmm.start()
diff --git a/jd/soulium.py b/auto/soulium.py
similarity index 96%
rename from jd/soulium.py
rename to auto/soulium.py
index e25a78c..80ee160 100644
--- a/jd/soulium.py
+++ b/auto/soulium.py
@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+# @author : jared
+# @date : 2018/11/10 22:48
import time
from appium import webdriver
from selenium.webdriver.common.by import By
diff --git a/jd/out.py b/auto/soulout.py
similarity index 97%
rename from jd/out.py
rename to auto/soulout.py
index c2cc17a..4b8eac1 100644
--- a/jd/out.py
+++ b/auto/soulout.py
@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+# @author : jared
+# @date : 2018/11/10 22:49
import json
import redis
import requests
@@ -6,7 +9,7 @@
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, String, Integer, TIMESTAMP, FLOAT, create_engine
-# mitmdump -p 8889 -s out.py
+# mitmdump -p 8889 -s soulout.py
ak = 'TCLfUCrFQDLWQrzz3NKYBwb8ZY57tgAt'
api2 = 'https://api.map.baidu.com/geocoder/v2/?location={},{}&ak={}&output=json'
diff --git a/blogs/README.md b/blogs/README.md
index 61d3aa1..ef60c22 100644
--- a/blogs/README.md
+++ b/blogs/README.md
@@ -1,3 +1,3 @@
-#### jianshu: 简书数据统计
+### 简书数据统计
The Statistics of jianshu
\ No newline at end of file
diff --git a/blogs/read.py b/blogs/read.py
index 6b0e911..1df643a 100644
--- a/blogs/read.py
+++ b/blogs/read.py
@@ -1,8 +1,8 @@
-import math
import time
+
+import math
import numpy
import requests
-import multiprocessing
from bs4 import BeautifulSoup
@@ -52,10 +52,10 @@ def count(self):
url = 'https://www.jianshu.com/u/{}'.format(self.uid)
resp = requests.get(url, headers=self.headers)
if resp.status_code == 200:
- bs = BeautifulSoup(resp.content, 'html.parser', from_encoding='UTF-8')
+ bs = BeautifulSoup(resp.text, 'html.parser')
avatar = bs.find(class_='avatar') # 头像
- self.avatar = 'https:' + avatar.img['src']
+ self.avatar = avatar.img['src']
nickname = bs.find(class_='name') # 昵称
self.nickname = nickname.text
diff --git a/blogs/start.sh b/blogs/start.sh
new file mode 100644
index 0000000..1d3e57e
--- /dev/null
+++ b/blogs/start.sh
@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+nohup python3 main.py 7999 >> /dev/null 2>&1 &
\ No newline at end of file
diff --git a/blogs/templates/show.html b/blogs/templates/show.html
index 27200aa..5f7d89d 100644
--- a/blogs/templates/show.html
+++ b/blogs/templates/show.html
@@ -1,4 +1,4 @@
-$def with (read_count)
+$def with (rc)
@@ -6,21 +6,22 @@
- 简书数据统计
-
+ happyJared - 简统
+
-
@@ -33,30 +34,30 @@
-$if read_count.exit:
+$if rc.exit:
-
-
rc.nickname
+
+
$rc.nickname
- 关注 : rc.watch 人 -
- 粉丝 : rc.fans 人 -
- 文章 : rc.articles 篇
- 写作 :
rc.words 字 -
- 收获喜欢 :
rc.likes 次 -
- 总阅读量 :
rc.total_reading 次 -
- 查询总耗时 ：
rc.time 秒
+ 关注 :
$rc.watch 人 -
+ 粉丝 :
$rc.fans 人 -
+ 文章 :
$rc.articles 篇
+
+ 写作 :
$rc.words 字 -
+ 收获喜欢 :
$rc.likes 次 -
+ 总阅读量 :
$rc.total_reading 次 -
+ 查询总耗时 ：
$rc.time 秒
404
-
- 用户不存在 , 您可以尝试下搜下面这些人
+ 404
+ 您可以尝试下搜下面这些人
简叔
+ 简黛玉
+ 彭小六
+ happyJared
diff --git a/drivers/chromedriver.exe b/drivers/chromedriver.exe
deleted file mode 100644
index 404f36a..0000000
Binary files a/drivers/chromedriver.exe and /dev/null differ
diff --git a/drivers/phantomjs.exe b/drivers/phantomjs.exe
deleted file mode 100644
index c644886..0000000
Binary files a/drivers/phantomjs.exe and /dev/null differ
diff --git a/logger/log.py b/logger/log.py
index 087988e..4422c59 100644
--- a/logger/log.py
+++ b/logger/log.py
@@ -5,21 +5,20 @@
import logging.config
-class Logger(object):
- def __init__(self):
+class Logger:
+ def __init__(self, log_name='info.log'):
"""日志相关配置"""
path = 'logs'
if not os.path.exists(path):
os.mkdir(path)
- path = '../config/logging.yml'
+ path = os.path.dirname(__file__) + '/logging.yml'
if os.path.exists(path):
with open(path, 'r', encoding='utf-8') as f:
- config = yaml.load(f)
+ config = yaml.safe_load(f)
logging.config.dictConfig(config)
+ logging.basicConfig(filename=log_name)
else:
- logging.basicConfig(level='INFO', filename='info.log',
+ logging.basicConfig(level='INFO', filename=log_name,
format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s')
-
- logging.info(">>> Load logger config")
diff --git a/config/logging.yml b/logger/logging.yml
similarity index 100%
rename from config/logging.yml
rename to logger/logging.yml
diff --git a/myflask/flask-01 b/myflask/flask-01
deleted file mode 100644
index 27c25f1..0000000
--- a/myflask/flask-01
+++ /dev/null
@@ -1,12 +0,0 @@
-from flask import Flask
-
-app = Flask(__name__)
-
-
-@app.route('/')
-def hello_world():
- return '2018 , Hello World !'
-
-
-if __name__ == '__main__':
- app.run(debug=True)
diff --git a/orm/sqlalchemy_crud.py b/orm/sqlalchemy_crud.py
index 8238d6b..50bd9e1 100644
--- a/orm/sqlalchemy_crud.py
+++ b/orm/sqlalchemy_crud.py
@@ -25,7 +25,7 @@ def __str__(self) -> str:
# 创建数据库连接（'数据库类型+数据库驱动名称://用户名:密码@ip地址:端口/数据库名'）
-conn = "postgresql+psycopg2://postgres:@:5432/postgres"
+conn = "postgresql+psycopg2://postgres:@:5432/postgres"
engine = create_engine(conn, encoding='UTF-8', echo=False) # echo=True表示输出执行日志,默认为False
# 删除映射数据表(如果存在)
diff --git a/scrapys/boss/boss/middlewares.py b/scrapys/boss/boss/middlewares.py
index 505ccf9..5b89b2a 100644
--- a/scrapys/boss/boss/middlewares.py
+++ b/scrapys/boss/boss/middlewares.py
@@ -5,15 +5,13 @@
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
-import time
import logging
import random
-import requests
-from utils import mycaptcha
+import time
+
from scrapy import signals
-from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
-from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
+from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
# from scrapys.nearjob import app
diff --git a/scrapys/boss/boss/spiders/job.py b/scrapys/boss/boss/spiders/job.py
index 754f2c4..2f243aa 100644
--- a/scrapys/boss/boss/spiders/job.py
+++ b/scrapys/boss/boss/spiders/job.py
@@ -1,4 +1,5 @@
# -*- coding: utf-8 -*-
+import re
import json
from urllib import parse
@@ -82,8 +83,9 @@ def parse_detail(self, response):
item['job_label'] = json.dumps(job_label, ensure_ascii=False)
job_desc = response.xpath('//div[@class="text"]/text()').extract()
item['job_description'] = '\n'.join(map(str.strip, job_desc))
- post_time = response.xpath('//span[@class="time"]/text()').extract_first().replace('发布于', '')
- item['post_job_time'] = mytime.str_to_date_with_format(post_time, '%Y-%m-%d %H:%M')
+ post_time = re.search(r'"upDate": "(.*)"', response.text)
+ post_time = post_time.group(1).replace("T", " ")
+ item['post_job_time'] = mytime.str_to_date(post_time)
address = response.xpath('//div[@class="location-address"]/text()').extract_first().replace(' ', '')
item['company_location'] = address
item['company_logo'] = response.xpath('//div[@class="info-company"]/div/a/img/@src').extract_first()
diff --git a/scrapys/lagou/lagou/settings.py b/scrapys/lagou/lagou/settings.py
index 4b4a856..e9173f9 100644
--- a/scrapys/lagou/lagou/settings.py
+++ b/scrapys/lagou/lagou/settings.py
@@ -22,12 +22,12 @@
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
-CONCURRENT_REQUESTS = 2
+CONCURRENT_REQUESTS = 1
# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
-DOWNLOAD_DELAY = 2
+DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
# CONCURRENT_REQUESTS_PER_DOMAIN = 16
# CONCURRENT_REQUESTS_PER_IP = 12
diff --git a/scrapys/lagou/lagou/spiders/job.py b/scrapys/lagou/lagou/spiders/job.py
index 18cf12b..8f71613 100644
--- a/scrapys/lagou/lagou/spiders/job.py
+++ b/scrapys/lagou/lagou/spiders/job.py
@@ -46,6 +46,7 @@ def start_requests(self):
def parse(self, response):
self.logger.warning(response)
resp = json.loads(response.body_as_unicode())
+ self.logger.info(resp)
success = resp.get('success')
self.logger.info('1. resp code %s success %s' % (resp.get('code'), success))
if success:
diff --git a/scrapys/lagou/lagou/spiders/job.sh b/scrapys/lagou/lagou/spiders/job.sh
index e96dc95..222b4af 100644
--- a/scrapys/lagou/lagou/spiders/job.sh
+++ b/scrapys/lagou/lagou/spiders/job.sh
@@ -1,3 +1,3 @@
#!/usr/bin/env bash
git pull origin master
-nohup python3 main.py >> /dev/null 2>&1 &
\ No newline at end of file
+nohup python3 main.py lagou >> /dev/null 2>&1 &
\ No newline at end of file
diff --git a/scrapys/lagou/lagou/spiders/main.py b/scrapys/lagou/lagou/spiders/main.py
index 8e9989f..ece44f0 100644
--- a/scrapys/lagou/lagou/spiders/main.py
+++ b/scrapys/lagou/lagou/spiders/main.py
@@ -15,7 +15,4 @@ def run():
if __name__ == '__main__':
- if len(sys.argv) == 1:
- cron.cron_blocking(job=run, day_of_week='4', hour='21')
- else:
- run()
+ run()
diff --git a/scrapys/together/together/spiders/yiqi.py b/scrapys/together/together/spiders/yiqi.py
index a4d783d..89432ce 100644
--- a/scrapys/together/together/spiders/yiqi.py
+++ b/scrapys/together/together/spiders/yiqi.py
@@ -13,15 +13,13 @@ class YiQiSpider(scrapy.Spider):
def __init__(self, name=None, **kwargs):
super().__init__(name, **kwargs)
self.getUserById = 'http://api.wondertech.com.cn/user/v2/users/getById'
- self.likeUser = 'http://api.wondertech.com.cn/fmvoice/v1/voice/vlike'
- self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiw' \
- 'iZXhwIjoxNTQwMDM1ODAxLCJ1c2VySWQiOjU4NjQzfQ.EADG6r551' \
- 'b_SKSfuOunJfuYD4LovgoPCNFnoIZ_VZxqiVEw8GTBcz1ix0zFhaDB' \
- 'PPTt3AtyDAp6tB5tAxewNaw'
+ # self.likeUser = 'http://api.wondertech.com.cn/fmvoice/v1/voice/vlike'
+ self.token = 'eyJhbGciOiJIUzUxMiJ9.eyJwaG9uZSI6IjEzNzA5NjQxNzEzIiwiZXhwIjoxNTU3MTIxMjQxLCJ1c2VySWQiOjU4NjQzfQ.lLvZd0L-ged9WdbQmj9ATS10l_GcyPLIR6P2-THjeZ9VA-iifxpfGBQAFtHPctN0aDHcyFWixiWVb5dzUn7JHw'
+
self.formData = {'token': self.token}
def start_requests(self):
- for user_id in range(305538, 305600):
+ for user_id in range(309130, 350000):
self.formData['id'] = str(user_id)
yield FormRequest(self.getUserById, formdata=self.formData,
callback=self.parse, meta={'uid': user_id})
@@ -36,19 +34,18 @@ def parse(self, response):
item = UserItem()
item['uid'] = uid
- sex = data.get('sex')
- item['sex'] = sex
- age = data.get('age')
- item['age'] = age
+ item['sex'] = data.get('sex')
+ item['age'] = data.get('age')
item['phone'] = data.get('phone')
item['nickname'] = data.get('nickName')
item['birthday'] = data.get('birthday')
head_pic = data.get('headPic')
item['head_pic'] = head_pic
+ if head_pic:
+ item['source'] = 'ios' if 'ios' in head_pic else 'android'
item['voice'] = data.get('voice')
item['available_voice'] = data.get('availableVoice')
- user_last_fm_voice = data.get('userLastFmVoice')
- item['user_last_fm_voice'] = user_last_fm_voice
+ item['user_last_fm_voice'] = data.get('userLastFmVoice')
item['region_code'] = data.get('regionCode')
info = data.get('regionInformation')
if info:
@@ -56,18 +53,9 @@ def parse(self, response):
if text:
item['region_information'] = ''.join(text)
item['last_app_version'] = data.get('lastAppVersion')
- if head_pic:
- item['source'] = 'ios' if 'ios' in head_pic else 'android'
item['create_time'] = data.get('createTime')
item['netease_accid'] = data.get('neteaseAccid')
item['netease_token'] = data.get('neteaseToken')
item['netease_status'] = data.get('neteaseStatus')
item['user_status'] = data.get('userStatus')
-
- region_information = item.get('region_information')
- if region_information:
- if sex == 2 and 20 <= age <= 23 and '广州' in region_information:
- form_data = {'token': self.token, 'voice': user_last_fm_voice, 'ou': uid}
- requests.post(self.likeUser, data=form_data)
-
yield item
diff --git a/single/planet.py b/single/planet.py
index 0f63c86..aa186f3 100644
--- a/single/planet.py
+++ b/single/planet.py
@@ -27,9 +27,7 @@ class Planet(object):
def __init__(self):
self.redis = Planet.redis
self.postgres = Planet.postgres
- self.__get_my_hash()
- self.__get_my_token()
- self.__get_my_user_id()
+ self.__get_my_info()
logging.info('Init my hash : %s and my user id : %s', Planet.my_hash, Planet.my_user_id)
def handler(self, sql, params):
@@ -42,32 +40,12 @@ def handler(self, sql, params):
return self.postgres.handler(sql, params)
- def __get_my_user_id(self):
- """获取userId值"""
-
- if not Planet.my_user_id:
- key = 'planet:my:uid4'
- Planet.my_user_id = self.redis.get(key)
- if not Planet.my_user_id:
- api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-status'
- resp = requests.post(api, json={}, headers=Planet.headers).json()
- Planet.my_user_id = resp['notification_settings'][0]['user_id']
- self.redis.set(key, Planet.my_user_id)
-
- def __get_my_hash(self):
- """获取hash值"""
-
- if not Planet.my_hash:
- key = 'planet:my:hash4'
- Planet.my_hash = self.redis.get(key)
- if not Planet.my_hash:
- api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-dog-hash'
- resp = requests.post(api, json={}, headers=Planet.headers).json()
- Planet.my_hash = resp['uid_hash']
- self.redis.set(key, Planet.my_hash)
-
- def __get_my_token(self):
- """获取token值"""
+ def __get_my_info(self):
+ """获取个人info"""
key = 'planet:my:token'
- Planet.headers['Authorization'] = self.redis.get(key)
+ Planet.headers['Authorization'] = "token " + self.redis.get(key)
+ api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/my-dog-hash'
+ resp = requests.post(api, json={}, headers=Planet.headers).json()
+ Planet.my_hash = resp['uid_hash']
+ Planet.my_user_id = resp['user_id']
diff --git a/single/planet_spider.py b/single/planet_spider.py
index 93ecb02..580ca72 100644
--- a/single/planet_spider.py
+++ b/single/planet_spider.py
@@ -39,7 +39,7 @@ def find_random_member(self):
logging.info("Start find random member")
api = 'https://www.quanquanyuanyuan.cn/huodong/dog/api/v2/dog-all-random'
for gender in range(1, 3):
- data = {"hash": Planet.my_hash, "pagesize": 50, "seed": 655572327, "gender": gender}
+ data = {"hash": Planet.my_hash, "pagesize": 50, "seed": 1333331304, "gender": gender}
logging.info("Start random member {}".format(gender))
for offset in range(PlanetSpider.max_size):
data["offset"] = offset
@@ -192,6 +192,6 @@ def __member_photos(self):
# 程序入口
if __name__ == '__main__':
ps = PlanetSpider()
- ps.find_random_member()
+ ps.find_nearby_member()
# 定时随机爬取
# cron.cron_blocking(job=ps.find_random_member, day_of_week='0-6', hour='2')
diff --git a/baike/README.md b/spider/baike/README.md
similarity index 100%
rename from baike/README.md
rename to spider/baike/README.md
diff --git a/baike/spider_download.py b/spider/baike/spider_download.py
similarity index 100%
rename from baike/spider_download.py
rename to spider/baike/spider_download.py
diff --git a/baike/spider_main.py b/spider/baike/spider_main.py
similarity index 100%
rename from baike/spider_main.py
rename to spider/baike/spider_main.py
diff --git a/baike/spider_output.py b/spider/baike/spider_output.py
similarity index 100%
rename from baike/spider_output.py
rename to spider/baike/spider_output.py
diff --git a/baike/spider_parser.py b/spider/baike/spider_parser.py
similarity index 100%
rename from baike/spider_parser.py
rename to spider/baike/spider_parser.py
diff --git a/baike/spider_url.py b/spider/baike/spider_url.py
similarity index 100%
rename from baike/spider_url.py
rename to spider/baike/spider_url.py
diff --git a/spider/bihu/robot.py b/spider/bihu/robot.py
new file mode 100644
index 0000000..4b968eb
--- /dev/null
+++ b/spider/bihu/robot.py
@@ -0,0 +1,126 @@
+import random
+import sys
+import time
+
+from bs4 import BeautifulSoup
+from selenium import webdriver
+from selenium.webdriver.chrome.options import Options
+
+argv = sys.argv
+argv_length = len(argv)
+
+
+def sleep(min_seconds=3, max_seconds=10):
+ """随机休眠"""
+ time.sleep(random.randint(min_seconds, max_seconds))
+
+
+# 无头模式
+options = Options()
+options.add_argument('--headless')
+options.add_argument('log-level=3')
+options.add_argument('--no-sandbox')
+options.add_argument('--disable-gpu')
+options.add_argument('window-size=1366x728')
+options.add_argument('--disable-dev-shm-usage')
+driver = webdriver.Chrome(chrome_options=options)
+
+# 登录币乎
+bihu = "https://bihu.com"
+bihu_login = "{}/login".format(bihu)
+driver.get(bihu_login)
+
+driver.maximize_window()
+
+username = argv[1] if argv_length > 1 else input("请输入用户名\n")
+driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[1]/input').send_keys(
+ username)
+password = argv[2] if argv_length > 2 else input("请输入登录密码\n")
+driver.find_element_by_xpath('//*[@id="root"]/div/div/div/div[1]/div[3]/div/div[1]/form/div[2]/input').send_keys(
+ password)
+
+driver.find_element_by_class_name("LoaderButton").click()
+sleep()
+
+# 登录成功
+print(driver.current_url)
+if driver.current_url == bihu + "/":
+
+ # 1. 点赞长文
+ bs = BeautifulSoup(driver.page_source, 'html.parser')
+ item_list = bs.find_all("li", class_="ArticleItem")
+ articles = {}
+ for article_item in item_list:
+ # 解析ArticleItem
+ user_info = article_item.find("div", class_="user-info")
+ p_list = user_info.find_all("p")
+ publish_time = p_list[-1].text
+
+ bottom = article_item.find("div", class_="item-bottom")
+ bottom_list = bottom.find_all("span")
+ # Key数 && 点赞数 && 评论数
+ key, like, comment = bottom_list
+ key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip()
+
+ content_info = article_item.find("div", class_="content-info")
+ info = content_info.find("a")
+ articles[info["href"]] = int(like_num)
+
+ num = 0
+ # 点赞操作
+ for key, value in sorted(articles.items(), key=lambda item: item[1]):
+ if num >= 6: break
+
+ article = "{}{}".format(bihu, key)
+ driver.get(article)
+ sleep()
+ target = driver.find_element_by_class_name("collect-div")
+ driver.execute_script("arguments[0].scrollIntoView()", target)
+ sleep()
+
+ details = driver.find_element_by_class_name("article-details-center2")
+ buttons = details.find_elements_by_tag_name("button")
+ if len(buttons) == 3:
+ num += 1
+ buttons[1].click()
+ sleep()
+
+ print("长文已点赞完...")
+
+ # 2. 点赞微文
+ driver.get("https://bihu.com/shortcontents")
+ sleep()
+
+ i = 0
+ while True:
+ short_items = driver.find_elements_by_class_name("ShortItem")
+ short_items_length = len(short_items)
+ for index, short_item in enumerate(short_items):
+ bottom = short_item.find_element_by_class_name("item-bottom")
+ buttons = bottom.find_elements_by_tag_name("button")
+ like_button = buttons[1]
+ if like_button and like_button.is_enabled():
+ like_button.click()
+ sleep(1, 3)
+ i += 1
+ if index + 1 == short_items_length:
+ sleep(5, 10)
+ if i >= 100: break
+
+ # bs = BeautifulSoup(driver.page_source, 'html.parser')
+ # item_list = bs.find_all("li", class_="ShortItem")
+ # num = 0
+ # for article_item in item_list:
+ # if num >= 100: break
+ # # 解析ShortItem
+ # user_info = article_item.find("div", class_="user-info")
+ # p_list = user_info.find_all("p")
+ # publish_time = p_list[-1].text
+ #
+ # bottom = article_item.find("div", class_="item-bottom")
+ # bottom_list = bottom.find_all("span")
+ # # Key数 && 点赞数 && 评论数
+ # key, like, comment = bottom_list
+ # key_num, like_num, comment_num = key.text.strip(), like.text.strip(), comment.text.strip()
+
+ print("微文已点赞完...")
diff --git a/freeproxy/app.py b/spider/freeproxy/app.py
similarity index 100%
rename from freeproxy/app.py
rename to spider/freeproxy/app.py
diff --git a/freeproxy/detect.py b/spider/freeproxy/detect.py
similarity index 100%
rename from freeproxy/detect.py
rename to spider/freeproxy/detect.py
diff --git a/freeproxy/detect.sh b/spider/freeproxy/detect.sh
similarity index 100%
rename from freeproxy/detect.sh
rename to spider/freeproxy/detect.sh
diff --git a/freeproxy/httpbin.py b/spider/freeproxy/httpbin.py
similarity index 100%
rename from freeproxy/httpbin.py
rename to spider/freeproxy/httpbin.py
diff --git a/freeproxy/kuai.py b/spider/freeproxy/kuai.py
similarity index 100%
rename from freeproxy/kuai.py
rename to spider/freeproxy/kuai.py
diff --git a/freeproxy/kuai.sh b/spider/freeproxy/kuai.sh
similarity index 100%
rename from freeproxy/kuai.sh
rename to spider/freeproxy/kuai.sh
diff --git a/freeproxy/xc.py b/spider/freeproxy/xc.py
similarity index 100%
rename from freeproxy/xc.py
rename to spider/freeproxy/xc.py
diff --git a/freeproxy/xc.sh b/spider/freeproxy/xc.sh
similarity index 100%
rename from freeproxy/xc.sh
rename to spider/freeproxy/xc.sh
diff --git a/tsflow/tsflow01.py b/tsflow/tsflow01.py
deleted file mode 100644
index b095ce1..0000000
--- a/tsflow/tsflow01.py
+++ /dev/null
@@ -1,8 +0,0 @@
-import tensorflow as tf
-
-hello = tf.constant('hello', dtype=tf.string)
-
-with tf.Session() as session:
- res = session.run(hello).decode('UTF-8')
- print(res)
- print(type(res))
diff --git a/utils/__init__.py b/utils/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/utils/vercode.py b/utils/vercode.py
deleted file mode 100644
index 97265fb..0000000
--- a/utils/vercode.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from PIL import Image
-import pytesseract
-
-pytesseract.pytesseract.tesseract_cmd = "E:\\jared\\Python\\Tesseract-OCR\\tesseract.exe"
-
-
-def recognize_code_image(image_file):
- # 打开图片
- image = Image.open(image_file)
- # image.show()
- # 转为灰度图片
- image_grey = image.convert('L')
- # image_grey.show()
- # 二值化处理
- table = []
- for i in range(256):
- if i < 140:
- table.append(0)
- else:
- table.append(1)
- image_bi = image_grey.point(table, '1')
- # image_bi.show()
- image_bi.save('C:\\Users\\CFL\\Desktop\\captcha1.png')
- # 识别验证码
- verify_code = pytesseract.image_to_string(image_bi)
-
- return verify_code
-
-
-if __name__ == '__main__':
- print(recognize_code_image('C:\\Users\\CFL\\Desktop\\captcha.png'))
diff --git a/wechat/__init__.py b/wechat/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/wechat/red_envelope.py b/wechat/red_envelope.py
new file mode 100644
index 0000000..5470cd4
--- /dev/null
+++ b/wechat/red_envelope.py
@@ -0,0 +1,22 @@
+import itchat
+from itchat.content import NOTE
+
+
+@itchat.msg_register(NOTE, isGroupChat=True)
+def receive_red_packet(msg):
+ content = msg['Content']
+ print(content)
+ if "收到红包" in content or '收到利是' in content:
+ groups = itchat.get_chatrooms(update=True)
+ chatroom = itchat.search_chatrooms(name='9012')
+ user_name = chatroom[0]['UserName']
+ group_name = ''
+ for group in groups:
+ if msg['FromUserName'] == group['UserName']:
+ group_name = group['NickName']
+ msg = 'Tip: 群 “{}” 有人正在发红包...'.format(group_name)
+ itchat.send(msg, toUserName=user_name)
+
+
+itchat.auto_login(hotReload=True, enableCmdQR=2)
+itchat.run()
diff --git a/wechat/red_envelope.sh b/wechat/red_envelope.sh
new file mode 100644
index 0000000..4339cca
--- /dev/null
+++ b/wechat/red_envelope.sh
@@ -0,0 +1 @@
+nohup python3 red_envelope.py >> /dev/null 2>&1 &
\ No newline at end of file
diff --git a/wechat/test.py b/wechat/test.py
deleted file mode 100644
index e69de29..0000000
diff --git a/wechat/tip.py b/wechat/tip.py
new file mode 100644
index 0000000..a66041a
--- /dev/null
+++ b/wechat/tip.py
@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# @author : Jared
+# @date : 2019/6/22 17:39
+
+import json
+import time
+from datetime import datetime
+
+import itchat
+import requests
+
+
+class CheLaiLe:
+ count, step, max_count = 0, 1, 90
+
+
+class ShellIsland:
+ start = False
+
+
+file_helper = 'filehelper'
+
+
+@itchat.msg_register(itchat.content.TEXT)
+def text(msg):
+ receive_text = msg['Text']
+ to_user_id = msg['ToUserName'] # 接收人
+ # from_user_id = msg['FromUserName'] # 发送人
+
+ if to_user_id == file_helper:
+ if receive_text.startswith("b"):
+ # 车来了
+ CheLaiLe.count = 0
+ while CheLaiLe.count <= CheLaiLe.max_count:
+ CheLaiLe.count += CheLaiLe.step
+ if datetime.now().hour < 12:
+ resp = requests.get(
+ 'https://api.chelaile.net.cn/bus/line!busesDetail.action'
+ '?sign=3BszHrCL%2BYb8Ix6to1kyJw%3D%3D&v=3.79.2&s=android'
+ '&cityId=040&targetOrder=6&lineId=020-07730-0').text
+ else:
+ resp = requests.get(
+ 'https://api.chelaile.net.cn/bus/line!busesDetail.action'
+ '?sign=CMkEe%2BVN716ghdReiM5G9Q%3D%3D&s=android&v=3.79.2'
+ '&cityId=040&targetOrder=4&lineId=020-07730-1').text
+ resp = resp.replace("**YGKJ", "").replace("YGKJ##", "")
+ json_content = json.loads(resp)
+ data = json_content.get('jsonr').get("data")
+ tip_desc, pre_arrival_desc = data.get("tip").get("desc"), data.get("depDesc")
+
+ send_msg = "{}\n[{}]".format(tip_desc, pre_arrival_desc)
+ itchat.send(send_msg, toUserName=file_helper)
+ time.sleep(30)
+
+ if receive_text.startswith("s"):
+ # 贝壳小岛
+ ShellIsland.start = not ShellIsland.start
+ while ShellIsland.start:
+ itchat.send("", toUserName=file_helper)
+ time.sleep(60)
+
+
+itchat.auto_login(hotReload=True, enableCmdQR=2)
+itchat.run()
diff --git a/wechat/tipjob.py b/wechat/tipjob.py
new file mode 100644
index 0000000..15b9cbe
--- /dev/null
+++ b/wechat/tipjob.py
@@ -0,0 +1,21 @@
+import os
+from apscheduler.schedulers.background import *
+
+
+def job_task():
+ scheduler = BlockingScheduler()
+ scheduler.add_job(tip, 'cron', hour=19, minute="30,45,59")
+ scheduler.start()
+
+
+def tip():
+ resp = os.popen("ps -ef|grep python | grep tip.py | wc -l").read()
+ if int(resp) <= 1:
+ print("脚本即将运行...")
+ os.system("nohup python3 tip.py >> /dev/null 2>&1 &")
+ else:
+ print("脚本已经运行...")
+
+
+if __name__ == '__main__':
+ job_task()
diff --git a/wechat/wx_itchat.py b/wechat/wx_itchat.py
index 59f83d9..44c74fe 100644
--- a/wechat/wx_itchat.py
+++ b/wechat/wx_itchat.py
@@ -138,7 +138,7 @@ def _handle_province(self, member_count):
for province in list(self.num_of_province.keys()):
number = self.num_of_province[province]
- if number / member_count < .02 and province != self.unknown_province:
+ if number / member_count < .01 and province != self.unknown_province:
other_province_num = self.num_of_province.get(self.unknown_province)
self.num_of_province.__setitem__(self.unknown_province, other_province_num + number)
del self.num_of_province[province]
@@ -150,6 +150,9 @@ def _plt_gender_bar(self, title):
# font size include: ‘xx-small’,x-small’,'small’,'medium’,‘large’,‘x-large’,‘xx-large’ or number
# plt.xticks(size='small',rotation=30)
+
+ plt.rcParams["font.sans-serif"] = ["SimHei"]
+ plt.rcParams["axes.unicode_minus"] = False
plt.bar('男', self.male_num, color='yellow')
plt.bar('女', self.female_num, color='pink')
plt.bar('未知', self.unknown_gender, color='gray')
@@ -166,6 +169,8 @@ def _plt_province_pie(self, title):
data = np.array(list(self.num_of_province.values()))
labels = list(self.num_of_province.keys())
+ plt.rcParams["font.sans-serif"] = ["SimHei"]
+ plt.rcParams["axes.unicode_minus"] = False
plt.pie(data, labels=labels, autopct='%.1f%%', )
plt.axis('equal')
plt.legend(loc=2, prop={'size': 5.5})
diff --git a/wechat/wx_mps.py b/wechat/wx_mps.py
index aa8a42a..fdd4360 100644
--- a/wechat/wx_mps.py
+++ b/wechat/wx_mps.py
@@ -1,69 +1,147 @@
-import json
import re
import time
+import json
+import random
+import urllib3
+import requests
+from bs4 import BeautifulSoup
from datetime import datetime
-import requests
+from utils import pgs, es
-from utils import pgs
+urllib3.disable_warnings()
+
+wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的)
+postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps)
+elastic = es.Es(host='localhost', port=12900, index='mp', doc='article')
+
+
+def load():
+ for i in [13, 14, 25, 3]:
+ sql = "select id,mps_biz,last_msg_id,app_msg_token,pass_ticket,wap_sid2 from " \
+ "tb_mps where id = {} and show = True".format(i)
+ result = postgres.fetch_all(sql)
+ for r in result:
+ r_id, r_mps_biz, r_last_msg_id, r_app_msg_token, r_pass_ticket, r_wap_sid2 = r[0], r[1], r[2], \
+ r[3], r[4], r[5]
+ r_cookie = 'wxuin=1604513290; version=62060619; lang=zh_TW; pass_ticket={}; wap_sid2={}'.format(
+ r_pass_ticket, r_wap_sid2)
+ mps = WxMps(r_id, r_mps_biz, r_pass_ticket, r_app_msg_token, r_cookie, r_last_msg_id)
+ mps.start()
class WxMps(object):
"""微信公众号文章、评论抓取爬虫"""
- def __init__(self, _biz, _pass_ticket, _app_msg_token, _cookie, _offset=0):
+ def __init__(self, _mps_id, _biz, _pass_ticket, _app_msg_token, _cookie, last_msg_id=0, _offset=0):
self.offset = _offset
+ self.mps_id = _mps_id
+ self.last_msg_id = last_msg_id # 上次抓取的最后消息的id
self.biz = _biz # 公众号标志
self.msg_token = _app_msg_token # 票据(非固定)
self.pass_ticket = _pass_ticket # 票据(非固定)
self.headers = {
'Cookie': _cookie, # Cookie(非固定)
- 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 '
+ 'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132'
}
- wx_mps = 'wxmps' # 这里数据库、用户、密码一致(需替换成实际的)
- self.postgres = pgs.Pgs(host='localhost', port='12432', db_name=wx_mps, user=wx_mps, password=wx_mps)
def start(self):
"""请求获取公众号的文章接口"""
+ start = True
offset = self.offset
- while True:
+ while start:
api = 'https://mp.weixin.qq.com/mp/profile_ext?action=getmsg&__biz={0}&f=json&offset={1}' \
'&count=10&is_ok=1&scene=124&uin=777&key=777&pass_ticket={2}&wxtoken=&appmsg_token' \
'={3}&x5=1&f=json'.format(self.biz, offset, self.pass_ticket, self.msg_token)
- resp = requests.get(api, headers=self.headers).json()
+ resp = requests.get(api, headers=self.headers, verify=False).json()
ret, status = resp.get('ret'), resp.get('errmsg') # 状态信息
- if ret == 0 or status == 'ok':
- print('Crawl article: ' + api)
- offset = resp['next_offset'] # 下一次请求偏移量
+ if 0 == ret or 'ok' == status:
+ # print('Crawl article: ' + api)
+
general_msg_list = resp['general_msg_list']
msg_list = json.loads(general_msg_list)['list'] # 获取文章列表
for msg in msg_list:
comm_msg_info = msg['comm_msg_info'] # 该数据是本次推送多篇文章公共的
msg_id = comm_msg_info['id'] # 文章id
+ if msg_id == self.last_msg_id:
+ print("最新一条啦{}".format(self.last_msg_id))
+ start = False
+ break
post_time = datetime.fromtimestamp(comm_msg_info['datetime']) # 发布时间
- # msg_type = comm_msg_info['type'] # 文章类型
+ msg_type = comm_msg_info['type'] # 文章类型
# msg_data = json.dumps(comm_msg_info, ensure_ascii=False) # msg原数据
- app_msg_ext_info = msg.get('app_msg_ext_info') # article原数据
- if app_msg_ext_info:
- # 本次推送的首条文章
- self._parse_articles(app_msg_ext_info, msg_id, post_time)
- # 本次推送的其余文章
- multi_app_msg_item_list = app_msg_ext_info.get('multi_app_msg_item_list')
- if multi_app_msg_item_list:
- for item in multi_app_msg_item_list:
- msg_id = item['fileid'] # 文章id
- if msg_id == 0:
- msg_id = int(time.time() * 1000) # 设置唯一id,解决部分文章id=0出现唯一索引冲突的情况
- self._parse_articles(item, msg_id, post_time)
- print('next offset is %d' % offset)
- else:
- print('Before break , Current offset is %d' % offset)
+ if 49 == msg_type:
+ # 图文消息
+ app_msg_ext_info = msg.get('app_msg_ext_info') # article原数据
+ if app_msg_ext_info:
+ # 本次推送的首条文章
+ self._parse_articles(app_msg_ext_info, msg_id, post_time, msg_type)
+ # 本次推送的其余文章
+ multi_app_msg_item_list = app_msg_ext_info.get('multi_app_msg_item_list')
+ if multi_app_msg_item_list:
+ for item in multi_app_msg_item_list:
+ msg_id = item['fileid'] # 文章id
+ if msg_id or not isinstance(msg_id, int):
+ msg_id = int(time.time()) # 设置唯一id,解决部分文章id=0出现唯一索引冲突的情况
+ self._parse_articles(item, msg_id, post_time, msg_type)
+ elif 1 == msg_type:
+ # 文字消息
+ content = comm_msg_info.get('content')
+ if content:
+ self._save_text_and_image(msg_id, post_time, msg_type, digest=content)
+ elif 3 == msg_type:
+ # 图片消息
+ image_msg_ext_info = msg.get('image_msg_ext_info')
+ cdn_url = image_msg_ext_info.get('cdn_url')
+ if cdn_url:
+ self._save_text_and_image(msg_id, post_time, msg_type, cover=cdn_url)
+
+ # 0：结束；1：继续
+ can_msg_continue = resp.get('can_msg_continue')
+ if not can_msg_continue:
+ print('Break , Current offset : %d' % offset)
break
+ offset = resp.get('next_offset') # 下一次请求偏移量
+ print('Next offset : %d' % offset)
+
+ def _save_es(self, json_data, article_id):
+ elastic.put_data(data_body=json_data, _id=article_id)
+
+ def _save_text_and_image(self, msg_id, post_time, msg_type, cover=None, digest=None):
+ """保存只是文字或图片消息"""
- def _parse_articles(self, info, msg_id, post_time):
+ article_id = postgres.handler(self._save_only_article(), (
+ msg_id, cover, digest, post_time, datetime.now(), self.mps_id, msg_type), fetch=True)
+
+ if article_id:
+ json_data = {"articleId": article_id, "cover": cover, "digest": digest, "mpsId": self.mps_id,
+ "msgId": msg_id, "postTime": post_time, "msgType": msg_type}
+ self._save_es(json_data, article_id)
+
+ @staticmethod
+ def crawl_article_content(content_url):
+ """抓取文章内容
+ :param content_url: 文章地址
+ """
+
+ try:
+ html = requests.get(content_url, verify=False).text
+ except:
+ print(content_url)
+ pass
+ else:
+ bs = BeautifulSoup(html, 'html.parser')
+ js_content = bs.find(id='js_content')
+ if js_content:
+ p_list = js_content.find_all('p')
+ content_list = list(map(lambda p: p.text, filter(lambda p: p.text != '', p_list)))
+ content = ''.join(content_list)
+ return content
+
+ def _parse_articles(self, info, msg_id, post_time, msg_type):
"""解析嵌套文章数据并保存入库"""
title = info.get('title') # 标题
@@ -75,21 +153,29 @@ def _parse_articles(self, info, msg_id, post_time):
# ext_data = json.dumps(info, ensure_ascii=False) # 原始数据
content_url = content_url.replace('amp;', '').replace('#wechat_redirect', '').replace('http', 'https')
- article_id = self.postgres.handler(self._save_article(), (msg_id, title, author, cover, digest,
- source_url, content_url, post_time,
- datetime.now()), fetch=True)
+ content = self.crawl_article_content(content_url)
+ article_id = postgres.handler(self._save_article(), (msg_id, title, author, cover, digest, source_url,
+ content_url, post_time, datetime.now(),
+ self.mps_id, content, msg_type), fetch=True)
if article_id:
+ json_data = {"articleId": article_id, "author": author, "content": content,
+ "contentURL": content_url, "cover": cover, "digest": digest,
+ "mpsId": self.mps_id, "msgId": msg_id, "postTime": post_time,
+ "sourceURL": source_url, "title": title, "msgType": msg_type}
+ self._save_es(json_data, article_id)
+ time.sleep(random.randint(2, 3))
self._parse_article_detail(content_url, article_id)
def _parse_article_detail(self, content_url, article_id):
"""从文章页提取相关参数用于获取评论,article_id是已保存的文章id"""
try:
- html = requests.get(content_url, headers=self.headers).text
- except:
- print('获取评论失败' + content_url)
+ resp = requests.get(content_url, headers=self.headers, verify=False)
+ except Exception as e:
+ print('获取评论失败 {} {}'.format(article_id, content_url))
else:
# group(0) is current line
+ html = resp.text
str_comment = re.search(r'var comment_id = "(.*)" \|\| "(.*)" \* 1;', html)
str_msg = re.search(r"var appmsgid = '' \|\| '(.*)'\|\|", html)
str_token = re.search(r'window.appmsg_token = "(.*)";', html)
@@ -101,7 +187,7 @@ def _parse_article_detail(self, content_url, article_id):
# 缺一不可
if appmsg_token and app_msg_id and comment_id:
- print('Crawl article comments: ' + content_url)
+ print('Crawl article {} comments'.format(article_id))
self._crawl_comments(app_msg_id, comment_id, appmsg_token, article_id)
def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id):
@@ -112,41 +198,64 @@ def _crawl_comments(self, app_msg_id, comment_id, appmsg_token, article_id):
'&pass_ticket={3}&wxtoken=777&devicetype=android-26&clientversion=26060739' \
'&appmsg_token={4}&x5=1&f=json'.format(self.biz, app_msg_id, comment_id,
self.pass_ticket, appmsg_token)
- resp = requests.get(api, headers=self.headers).json()
- ret, status = resp['base_resp']['ret'], resp['base_resp']['errmsg']
- if ret == 0 or status == 'ok':
- elected_comment = resp['elected_comment']
- for comment in elected_comment:
- nick_name = comment.get('nick_name') # 昵称
- logo_url = comment.get('logo_url') # 头像
- comment_time = datetime.fromtimestamp(comment.get('create_time')) # 评论时间
- content = comment.get('content') # 评论内容
- content_id = comment.get('content_id') # id
- like_num = comment.get('like_num') # 点赞数
- # reply_list = comment.get('reply')['reply_list'] # 回复数据
-
- self.postgres.handler(self._save_article_comment(), (article_id, comment_id, nick_name, logo_url,
- content_id, content, like_num, comment_time,
- datetime.now()))
+ try:
+ resp = requests.get(api, headers=self.headers, verify=False).json()
+ except:
+ print('Article {} no Comment'.format(article_id))
+ else:
+ ret, status = resp['base_resp']['ret'], resp['base_resp']['errmsg']
+ if ret == 0 or status == 'ok':
+ elected_comment = resp['elected_comment']
+ for comment in elected_comment:
+ nick_name = comment.get('nick_name') # 昵称
+ logo_url = comment.get('logo_url') # 头像
+ comment_time = datetime.fromtimestamp(comment.get('create_time')) # 评论时间
+ content = comment.get('content') # 评论内容
+ content_id = comment.get('content_id') # id
+ like_num = comment.get('like_num') # 点赞数
+
+ reply_list = comment.get('reply')['reply_list'] # 回复数据
+ reply_content = reply_like_num = reply_create_time = None
+ if reply_list:
+ first_reply = reply_list[0]
+ reply_content = first_reply.get('content')
+ reply_like_num = first_reply.get('reply_like_num')
+ reply_create_time = datetime.fromtimestamp(first_reply.get('create_time'))
+
+ postgres.handler(self._save_article_comment(), (article_id, comment_id, nick_name, logo_url,
+ content_id, content, like_num, comment_time,
+ datetime.now(), reply_content, reply_like_num,
+ reply_create_time, self.mps_id))
+
+ @staticmethod
+ def _save_only_article():
+ sql = 'insert into tb_article(msg_id,cover,digest,post_time,create_time,mps_id,msg_type) ' \
+ 'values(%s,%s,%s,%s,%s,%s,%s) returning id'
+ return sql
@staticmethod
def _save_article():
- sql = 'insert into tb_article(msg_id,title,author,cover,digest,source_url,content_url,post_time,create_time) ' \
- 'values(%s,%s,%s,%s,%s,%s,%s,%s,%s) returning id'
+ sql = 'insert into tb_article(msg_id,title,author,cover,digest,source_url,content_url,post_time,create_time,' \
+ 'mps_id,content,msg_type) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) returning id'
return sql
@staticmethod
def _save_article_comment():
sql = 'insert into tb_article_comment(article_id,comment_id,nick_name,logo_url,content_id,content,like_num,' \
- 'comment_time,create_time) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)'
+ 'comment_time,create_time,reply_content,reply_like_num,reply_create_time,mps_id) ' \
+ 'values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)'
return sql
if __name__ == '__main__':
- biz = 'MzI2NDk5NzA0Mw==' # "36氪"
- pass_ticket = 'NDndxxaZ7p6Z9PYulWpLqMbI0i3ULFeCPIHBFu1sf5pX2IhkGfyxZ6b9JieSYRUy'
- app_msg_token = '971_ywDL%252FCnBNQJEyvFe1H1O5hAhJydL3CpqZwirxw~~'
- cookie = 'wap_sid2=CO3YwOQHEogBdENPSVdaS3pHOWc1V2QzY1NvZG9PYk1DMndPS3NfbGlHM0Vfal8zLU9kcUdkWTQxdUYwckFBT3RZM1VYUXFaWkFad3NVaWFXZ28zbEFIQ2pTa1lqZktfb01vcGdPLTQ0aGdJQ2xOSXoxTVFvNUg3SVpBMV9GRU1lbnotci1MWWl5d01BQUF+fjCj45PcBTgNQAE='
- # 以上信息不同公众号每次抓取都需要借助抓包工具做修改
- wxMps = WxMps(biz, pass_ticket, app_msg_token, cookie)
- wxMps.start() # 开始爬取文章及评论
+ # _id = 18
+ # biz = 'MjM5NjAxOTU4MA=='
+ # app_msg_token = '987_cjLuYj%2BqEZfnkrXFuo06pvB9JHXZHHgSqtFVjg~~'
+ # # 上面3个定义与公众号相关【_id仅为个人数据库中的一个标志】，下面3个跟当前微信会话有关
+ # pass_ticket = '4UNJTJN77aahj26LxFmo3IgBRs2kAqTpJn++Y/0stu3Q+nXVF8zb00nOMkRiFq5h'
+ # wap_sid2 = 'CIrci/0FElx1T2p1M2d6NFczNk1Dd05NUHRab0Fjb205ekxNSlBHV3VlNTVOYXl0Qk5ZUDc3Y2JsZ01jclczYWx3eXdMNUNoMDZhOUF4SkUxRHNIRHc3aG1SZExRZHNEQUFBfjDFjcfgBTgNQJVO'
+ # cookie = 'wxuin=1604513290; version=62060426; pass_ticket={}; wap_sid2={}'.format(pass_ticket, wap_sid2)
+ # wxMps = WxMps(_id, biz, pass_ticket, app_msg_token, cookie)
+ # # 开始爬取文章及评论
+ # wxMps.start()
+ load()
diff --git a/wechat/wx_mps.sh b/wechat/wx_mps.sh
index 317ffb9..b802e48 100644
--- a/wechat/wx_mps.sh
+++ b/wechat/wx_mps.sh
@@ -1,2 +1,2 @@
#!/usr/bin/env bash
-nohup python3 wx_mps.py >> /dev/null 2>&1 &
\ No newline at end of file
+nohup python3 wx_mps.py >> mps.log 2>&1 &
\ No newline at end of file