Skip to content

Commit 344e3f3

Browse files
author
Jared
committed
refactor jianshu
1 parent bfda94c commit 344e3f3

2 files changed

Lines changed: 50 additions & 55 deletions

File tree

spider/jianshu/jsloader.py

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
import os
33
from random import shuffle
44

5+
from utils import rds
6+
57
base = "D:\\Data\\"
68

79

@@ -61,3 +63,27 @@ def load_comment():
6163
shuffle(data)
6264
print("读取结束")
6365
return data
66+
67+
68+
def load_server():
69+
""" 加载服务 """
70+
71+
print("开始读取")
72+
with open('{}{}.csv'.format(base, "server"), "r", encoding='utf-8') as file:
73+
reader_data = csv.reader(file)
74+
for row in reader_data:
75+
alias, ip, port, password = row
76+
print("读取结束")
77+
return alias, ip, port, password
78+
79+
80+
def load_cookie():
81+
""" 加载cookie """
82+
83+
print("开始读取")
84+
alias, ip, port, password = load_server()
85+
redis = rds.Rds(host=ip, port=port, db=10, password=password).redis_cli
86+
cookie_data = redis.hgetall("jianshu")
87+
redis.close()
88+
print("读取结束")
89+
return cookie_data

spider/jianshu/robot.py

Lines changed: 24 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,31 @@
11
import logging
2-
import os
3-
import platform
42
import random
5-
import sys
3+
import getpass
64
import time
75

86
from selenium import webdriver
97
from selenium.webdriver.chrome.options import Options
108

11-
argv = sys.argv
12-
argv_length = len(argv)
13-
14-
username = argv[1] if argv_length > 1 else input("请输入登录账号\n")
15-
password = argv[2] if argv_length > 2 else input("请输入登录密码\n")
16-
role = int(argv[3]) if argv_length > 3 else input("请输入Role\n")
17-
18-
if platform.system() == "Linux":
19-
os.system("ps -ef | grep chrome | awk '{print $2}' | xargs kill -9")
9+
import jsloader
10+
from logger import log
2011

2112

2213
def sleep(min_seconds=3, max_seconds=10):
2314
"""随机休眠"""
2415
time.sleep(random.randint(min_seconds, max_seconds))
2516

2617

27-
logging.basicConfig(level='INFO', filename='robot.log',
28-
format='%(asctime)s %(filename)s[%(lineno)d] %(name)s (%(levelname)s): %(message)s')
29-
# 无头模式
30-
options = Options()
31-
options.add_argument('--headless')
32-
options.add_argument('log-level=3')
33-
options.add_argument('--no-sandbox')
34-
options.add_argument('--disable-gpu')
35-
options.add_argument('window-size=1366x728')
36-
options.add_argument('--disable-dev-shm-usage')
37-
options.add_argument('blink-settings=imagesEnabled=false')
38-
driver = webdriver.Chrome(chrome_options=options)
39-
40-
# 登录简书
18+
log.Logger('robot.log')
4119
jianshu = "https://www.jianshu.com"
42-
driver.get("{}/sign_in".format(jianshu))
43-
driver.maximize_window()
44-
45-
# 豆瓣登录
46-
driver.find_element_by_xpath("/html/body/div[1]/div[2]/div/div/ul/li[4]").click()
47-
driver.find_element_by_class_name("douban").click()
48-
sleep()
49-
50-
# 切换窗口
51-
driver.close()
52-
window_handles = driver.window_handles
53-
driver.switch_to.window(window_handles[-1])
54-
driver.find_element_by_id("inp-alias").send_keys(username)
55-
driver.find_element_by_id("inp-pwd").send_keys(password)
56-
57-
try:
58-
driver.find_element_by_xpath("/html/body/div/div[2]/form[2]/div[4]/input[1]").click()
59-
except:
60-
logging.error("{} : 登录失败".format(role))
61-
else:
62-
sleep()
63-
64-
page_title = driver.title
65-
if page_title.find("简书") == -1:
66-
logging.error("{} :{}".format(role, page_title))
67-
else:
20+
cookie_data = jsloader.load_cookie()
21+
for role, cookie in cookie_data.items():
22+
# 无头模式
23+
chrome_dir = r"--user-data-dir=C:\Users\{}\AppData\Local\Google\Chrome\User Data".format(getpass.getuser())
24+
options = Options()
25+
options.add_argument(chrome_dir)
26+
options.add_argument('--headless')
27+
driver = webdriver.Chrome(options=options)
28+
try:
6829
# 1. 收益
6930
driver.get('{}/mobile/fp?read_mode=night'.format(jianshu)), sleep(1, 2)
7031
elements = driver.find_elements_by_class_name("order")
@@ -95,6 +56,14 @@ def sleep(min_seconds=3, max_seconds=10):
9556
except:
9657
logging.error("Button_Element : {}".format(button_element))
9758
pass
98-
finally:
99-
driver.close()
100-
driver.quit()
59+
except:
60+
logging.error("{} : 异常".format(role))
61+
else:
62+
# 重新获取 cookie
63+
driver.add_cookie({"name": "remember_user_token",
64+
"value": ""})
65+
pass
66+
finally:
67+
driver.delete_all_cookies()
68+
driver.close()
69+
driver.quit()

0 commit comments

Comments
 (0)