Skip to content

Commit ff9e25e

Browse files
lzjun567lzjun567
committed
add weixin comment
1 parent 715b313 commit ff9e25e

14 files changed

Lines changed: 282 additions & 20 deletions

‎.DS_Store‎

2 KB
Binary file not shown.

‎pdf/crawler.py‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,35 +38,33 @@ class Crawler(object):
3838
def __init__(self, name, start_url):
3939
"""
4040
初始化
41-
:param name: 保存问的PDF文件名,不需要后缀名
41+
:param name: 将要被保存为PDF的文件名称
4242
:param start_url: 爬虫入口URL
4343
"""
4444
self.name = name
4545
self.start_url = start_url
4646
self.domain = '{uri.scheme}://{uri.netloc}'.format(uri=urlparse(self.start_url))
4747

48-
def crawl(self, url):
48+
@staticmethod
49+
def request(url, **kwargs):
4950
"""
50-
pass
51+
网络请求,返回response对象
5152
:return:
5253
"""
53-
print(url)
54-
response = requests.get(url)
54+
response = requests.get(url, **kwargs)
5555
return response
5656

5757
def parse_menu(self, response):
5858
"""
59-
解析目录结构,获取所有URL目录列表:由子类实现
60-
:param response 爬虫返回的response对象
61-
:return: url 可迭代对象(iterable) 列表,生成器,元组都可以
59+
从response中解析出所有目录的URL链接
6260
"""
6361
raise NotImplementedError
6462

6563
def parse_body(self, response):
6664
"""
6765
解析正文,由子类实现
6866
:param response: 爬虫返回的response对象
69-
:return: 返回经过处理的html文本
67+
:return: 返回经过处理的html正文文本
7068
"""
7169
raise NotImplementedError
7270

@@ -89,8 +87,8 @@ def run(self):
8987
'outline-depth': 10,
9088
}
9189
htmls = []
92-
for index, url in enumerate(self.parse_menu(self.crawl(self.start_url))):
93-
html = self.parse_body(self.crawl(url))
90+
for index, url in enumerate(self.parse_menu(self.request(self.start_url))):
91+
html = self.parse_body(self.request(url))
9492
f_name = ".".join([str(index), "html"])
9593
with open(f_name, 'wb') as f:
9694
f.write(html)

‎weixin/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# encoding: utf-8
2+
__author__ = 'liuzhijun'

‎weixin/db.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
from pymongo import MongoClient
2+
3+
4+
class Conn(object):
5+
client = MongoClient('localhost', 27017)
6+
db = client['weixin-comment']
7+
8+
@classmethod
9+
def insert_many(cls, data):
10+
cls.db['comments'].insert_many(data)
11+
12+
@classmethod
13+
def query(cls):
14+
data = cls.db['comments'].find()
15+
return data
16+
17+
18+
conn = Conn()

‎weixin/python-logo.png‎

22 KB
Loading

‎weixin/python之禅.jpg‎

33.8 KB
Loading

‎weixin/stopwords.txt‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
the
2+
of
3+
is
4+
and
5+
to
6+
in
7+
that
8+
we
9+
for
10+
an
11+
are
12+
by
13+
be
14+
as
15+
on
16+
with
17+
can
18+
if
19+
from
20+
which
21+
you
22+
it
23+
this
24+
then
25+
at
26+
have
27+
all
28+
not
29+
one
30+
has
31+
or
32+
that
33+
的
34+
了
35+
和
36+
是
37+
就
38+
都
39+
而
40+
及
41+
與
42+
著
43+
或
44+
什么
45+
还是
46+
就是
47+
还要
48+
可以
49+
没有
50+
看看
51+
怎么
52+
那么
53+
不能
54+
分享
55+
出来
56+
已经
57+
下载
58+
有点
59+
今天
60+
很多
61+
因为
62+
你们
63+
完全
64+
一次
65+
quot
66+
不是
67+
这样
68+
这么
69+
觉得
70+
知道
71+
只有
72+
不过
73+
需要
74+
还有
75+
一个
76+
这个
77+
回复
78+
现在
79+
不错
80+
大家
81+
应该
82+
我刚
83+
不会
84+
如果
85+
时候
86+
开始
87+
正在
88+
为啥
89+
各种
90+
一個
91+
沒有
92+
我們
93+
你們
94+
妳們
95+
他們
96+
她們
97+
是否
98+
留言

‎weixin/weixin.py‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
# encoding: utf-8
2+
3+
import json
4+
import re
5+
import time
6+
from http.cookies import SimpleCookie
7+
8+
import jieba.analyse
9+
import matplotlib.pyplot as plt
10+
import requests
11+
from scipy.misc import imread
12+
from wordcloud import WordCloud
13+
14+
from pymongo import MongoClient
15+
16+
17+
class Conn(object):
18+
client = MongoClient('localhost', 27017)
19+
db = client['weixin-comment']
20+
21+
@classmethod
22+
def insert_many(cls, data):
23+
cls.db['comments'].insert_many(data)
24+
25+
@classmethod
26+
def query(cls):
27+
data = cls.db['comments'].find()
28+
return data
29+
30+
31+
conn = Conn()
32+
33+
raw_cookie = """
34+
gsScrollPos-5517=; tvfe_boss_uuid=9c139f72f8ae693f; pac_uid=1_253421576; pgv_pvi=5182785536; RK=0IMfVbYuWK;
35+
"""
36+
37+
cookie = SimpleCookie(raw_cookie)
38+
requests_cookies = dict([(c, cookie[c].value) for c in cookie])
39+
40+
41+
def main():
42+
# 普通留言, 精选留言总数
43+
normal_count, selected_count = 141, 100
44+
# 普通留言url
45+
normal_url = "https://mp.weixin.qq.com/misc/appmsgcomment?" \
46+
"action=list_comment&" \
47+
"mp_version=7&" \
48+
"type=0&" \
49+
"comment_id=2881104117&" \
50+
"begin={begin}&" \
51+
"count=10&" \
52+
"token=1300595798&" \
53+
"lang=zh_CN"
54+
55+
# 精选留言url
56+
selected_url = "https://mp.weixin.qq.com/misc/appmsgcomment?action=list_comment&mp_version=7&type=1" \
57+
"&begin={begin}&count=10&comment_id=2881104117&token=1300595798&lang=zh_CN"
58+
59+
dd = dict([(normal_count, selected_url), (selected_count, normal_url)])
60+
61+
for k, v in dd.items():
62+
crawler(k, v)
63+
64+
65+
def crawler(count, url):
66+
for i in range(0, count, 10):
67+
r = requests.get(url.format(begin=i), cookies=requests_cookies)
68+
match = re.search(r'"comment":(\[\{.*\}\])', r.text, re.S)
69+
if match:
70+
data = json.loads(match.group(1), encoding="utf-8")
71+
conn.insert_many(data)
72+
time.sleep(1)
73+
74+
75+
def display():
76+
# 读取数据
77+
78+
data = conn.query()
79+
for c in data:
80+
yield c.get("content")
81+
82+
83+
def word_segment(texts):
84+
# 分词处理
85+
jieba.analyse.set_stop_words("./stopwords.txt")
86+
for text in texts:
87+
tags = jieba.analyse.extract_tags(text, topK=20)
88+
yield " ".join(tags)
89+
90+
91+
def generate_img(texts):
92+
# 生成词云图片
93+
data = " ".join(text for text in texts)
94+
95+
mask_img = imread('./python-logo.png', flatten=True)
96+
wordcloud = WordCloud(
97+
font_path='/Library/Fonts//华文黑体.ttf',
98+
background_color='white',
99+
mask=mask_img
100+
).generate(data)
101+
plt.imshow(wordcloud)
102+
plt.axis('off')
103+
plt.savefig('./wordcloud.jpg', dpi=600)
104+
105+
106+
if __name__ == '__main__':
107+
main()
108+
# 生成词云方法
109+
# generate_img(word_segment(display()))

‎weixin/wordcloud.jpg‎

1.51 MB
Loading

‎zhihu/auto_login.py‎

Lines changed: 16 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,15 @@
2020

2121
# 使用登录cookie信息
2222
session = requests.session()
23-
session.cookies = cookiejar.LWPCookieJar(filename='cookies')
24-
try:
25-
session.cookies.load(ignore_discard=True)
26-
except:
27-
print("还没有cookie信息")
23+
# session.cookies = cookiejar.LWPCookieJar(filename='cookies')
24+
# try:
25+
# session.cookies.load(ignore_discard=True)
26+
# except:
27+
# print("还没有cookie信息")
2828

2929

3030
def get_xsrf():
31-
response = session.get("https://www.zhihu.com", headers=headers)
31+
response = session.get("https://www.zhihu.com", headers=headers, verify=False)
3232
soup = BeautifulSoup(response.content, "html.parser")
3333
xsrf = soup.find('input', attrs={"name": "_xsrf"}).get("value")
3434
return xsrf
@@ -41,6 +41,7 @@ def get_captcha():
4141
"""
4242
t = str(int(time.time() * 1000))
4343
captcha_url = 'https://www.zhihu.com/captcha.gif?r=' + t + "&type=login"
44+
print(captcha_url)
4445
r = session.get(captcha_url, headers=headers)
4546
with open('captcha.jpg', 'wb') as f:
4647
f.write(r.content)
@@ -56,13 +57,19 @@ def login(email, password):
5657
'_xsrf': get_xsrf(),
5758
"captcha": get_captcha(),
5859
'remember_me': 'true'}
60+
print(session.cookies)
5961
response = session.post(login_url, data=data, headers=headers)
6062
login_code = response.json()
6163
print(login_code['msg'])
62-
session.cookies.save()
64+
print(session.cookies)
65+
r = session.get("https://www.zhihu.com/settings/profile", headers=headers)
66+
print(r.status_code)
67+
print(r.text)
68+
with open("xx.html", "wb") as f:
69+
f.write(r.content)
6370

6471

6572
if __name__ == '__main__':
66-
email = input('email：')
67-
password = input("password:")
73+
email = "lzjun567@qq.com"
74+
password = "lzjun854979"
6875
login(email, password)

0 commit comments

Comments
 (0)