Skip to content

Commit 6e43fb9

Browse files
committed
python3 compatible
1 parent 1fda2d4 commit 6e43fb9

4 files changed

Lines changed: 22 additions & 82 deletions

File tree

README.md

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ requests、beautifulsoup 是爬虫两大神器,reuqests 用于网络请求,b
66

77
```python
88
pip install requests
9-
pip install beautifulsoup
9+
pip install beautifulsoup4
1010
pip install pdfkit
1111
```
1212

@@ -26,10 +26,18 @@ python crawler.py
2626
### 效果图
2727
![image](./crawer-pdf.png)
2828

29-
>作者:liuzhijun
29+
### 常见问题
3030

31-
>微信号: lzjun567
31+
1. SyntaxError: Missing parentheses in call to 'print'
32+
33+
beautifulsoup3不支持python2,所以下载beautifulsoup是要指定 beautifusoup4
34+
2. 如果是使用PyCharm开发, 那么运行的时候要在shell/cmd 窗口执行脚本, 直接在Pycharm中运行会找不到 wkhtmltopdf命令
3235

36+
37+
### contact me
38+
39+
>作者:liuzhijun
40+
>微信号: lzjun567
3341
>公众号:一个程序员的微站(VTtalk)
3442
3543

crawler.py

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import os
33
import re
44
import time
5-
5+
import logging
66
import pdfkit
77
import requests
88
from bs4 import BeautifulSoup
@@ -30,7 +30,7 @@ def parse_url_to_html(url, name):
3030
"""
3131
try:
3232
response = requests.get(url)
33-
soup = BeautifulSoup(response.content, "html5lib")
33+
soup = BeautifulSoup(response.content, 'html.parser')
3434
# 正文
3535
body = soup.find_all(class_="x-wiki-content")[0]
3636
# 标题
@@ -47,17 +47,22 @@ def parse_url_to_html(url, name):
4747
pattern = "(<img .*?src=\")(.*?)(\")"
4848

4949
def func(m):
50-
rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)
51-
return rtn
50+
if not m.group(3).startswith("http"):
51+
rtn = m.group(1) + "http://www.liaoxuefeng.com" + m.group(2) + m.group(3)
52+
return rtn
53+
else:
54+
return m.group(1)+m.group(2)+m.group(3)
5255

5356
html = re.compile(pattern).sub(func, html)
5457
html = html_template.format(content=html)
58+
html = html.encode("utf-8")
5559
with open(name, 'wb') as f:
5660
f.write(html)
5761
return name
5862

5963
except Exception as e:
60-
print(e.message)
64+
65+
logging.error("解析错误", exc_info=True)
6166

6267

6368
def get_url_list():
@@ -66,7 +71,7 @@ def get_url_list():
6671
:return:
6772
"""
6873
response = requests.get("http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000")
69-
soup = BeautifulSoup(response.content, "html5lib")
74+
soup = BeautifulSoup(response.content, "html.parser")
7075
menu_tag = soup.find_all(class_="uk-nav uk-nav-side")[1]
7176
urls = []
7277
for li in menu_tag.find_all("li"):

file1.html

Lines changed: 0 additions & 36 deletions
This file was deleted.

file2.html

Lines changed: 0 additions & 37 deletions
This file was deleted.

0 commit comments

Comments
 (0)