Skip to content

Commit be88e4b

Browse files
author
Jaden Tseng
committed
增加功能：Python爬虫，爬取runoob网站上的教程，并转成pdf文件
1 parent a586700 commit be88e4b

7 files changed

Lines changed: 209 additions & 0 deletions

File tree

‎runoob2pdf/README.md‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#Python 爬虫：把runoob网站上的各类教程转换成 PDF 电子书
2+
3+
### 系统要求
4+
python3.4以上版本, 不支持python2.x
5+
6+
7+
### 准备工具
8+
9+
requests、beautifulsoup 是爬虫两大神器，reuqests 用于网络请求，beautifusoup 用于操作 html 数据。有了这两把梭子，干起活来利索。scrapy 这样的爬虫框架我们就不用了，这样的小程序派上它有点杀鸡用牛刀的意思。此外，既然是把 html 文件转为 pdf，那么也要有相应的库支持， wkhtmltopdf 就是一个非常的工具，它可以用适用于多平台的 html 到 pdf 的转换，pdfkit 是 wkhtmltopdf 的Python封装包。click是一款命令行工具参数工具，用于在命令行传递参数。
10+
11+
首先安装好下面的依赖包
12+
13+
```python
14+
pip install requests
15+
pip install beautifulsoup4
16+
pip install pdfkit
17+
pip install click
18+
```
19+
20+
### 安装 wkhtmltopdf
21+
Windows平台直接在 [http://wkhtmltopdf.org/downloads.html](http://wkhtmltopdf.org/downloads.html) 下载稳定版的 wkhtmltopdf 进行安装，安装完成之后把该程序的执行路径加入到系统环境 $PATH 变量中，否则 pdfkit 找不到 wkhtmltopdf 就出现错误 “No wkhtmltopdf executable found”。Ubuntu 和 CentOS 可以直接用命令行进行安装
22+
23+
```shell
24+
$ sudo apt-get install wkhtmltopdf # ubuntu
25+
$ sudo yum intsall wkhtmltopdf # centos
26+
```
27+
28+
### 运行
29+
```python
30+
python runoob2pdf.py
31+
```
32+
33+
### 说明
34+
执行 python runoob2pdf.py后
35+
会提示让你输入
36+
1. runoob网站上的教程主页地址，主页地址就是网页顶部菜单上对应的地址。
37+
如效果图。
38+
2. 输入保存的pdf文件名。
39+
40+
### 效果图
41+
![image](./runoob2pdf.jpg)
42+
![image](./runoob2pdf_1.jpg)
43+
![image](./runoob2pdf_2.jpg)
44+
![image](./runoob2pdf_3.jpg)
45+
46+
### 特别说明
47+
感谢《Python 爬虫：把廖雪峰的教程转换成 PDF 电子书》的作者liuzhijun，本项目的代码都是基于他的代码改动后实现。
48+
49+
### Contact me
50+
>作者：jadentseng
51+
>微信： cheney2010
52+
53+

‎runoob2pdf/__init__.py‎

Whitespace-only changes.

‎runoob2pdf/runoob2pdf.jpg‎

37.9 KB
Loading

‎runoob2pdf/runoob2pdf.py‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
# -*- coding=utf-8 -*-
2+
import os
3+
import re
4+
import time
5+
from urllib.parse import urlparse
6+
from bs4 import BeautifulSoup
7+
import pdfkit
8+
import requests
9+
10+
__author__ = 'jaden.tseng@foxmail.com'
11+
12+
import click
13+
14+
html_template = """
15+
<!DOCTYPE html>
16+
<html lang="en">
17+
<head>
18+
<meta charset="UTF-8">
19+
</head>
20+
<body>
21+
{content}
22+
</body>
23+
</html>
24+
25+
"""
26+
27+
28+
def get_url_list(url):
29+
"""
30+
获取所有URL目录列表
31+
:return:
32+
"""
33+
last_position = find_last(url, "/") + 1
34+
tutorial_url_head = url[0:last_position]
35+
domain = get_domain(url) + "/"
36+
37+
response = requests.get(url)
38+
soup = BeautifulSoup(response.content, "html.parser")
39+
menu_tag = soup.find(class_="design")
40+
urls = []
41+
for a in menu_tag.find_all("a"):
42+
href = str(a.get('href'))
43+
result = href.find('/')
44+
if result == -1:
45+
url = tutorial_url_head + href
46+
else:
47+
url = domain + href
48+
urls.append(url)
49+
return urls
50+
51+
52+
def parse_url_to_html(url, name):
53+
"""
54+
解析URL，返回HTML内容
55+
:param url:解析的url
56+
:param name: 保存的html文件名
57+
:return: html
58+
"""
59+
try:
60+
response = requests.get(url)
61+
soup = BeautifulSoup(response.content, 'html.parser')
62+
# 正文
63+
body = soup.find_all(class_="article-intro")
64+
# 标题
65+
# title = soup.find_all('h1')[1].get_text()
66+
67+
# 标题加入到正文的最前面，居中显示
68+
# center_tag = soup.new_tag("center")
69+
# title_tag = soup.new_tag('h1')
70+
# title_tag.string = title
71+
# center_tag.insert(1, title_tag)
72+
# body.insert(1, center_tag)
73+
h = str(body)
74+
html = h[1:-1]
75+
# body中的img标签的src相对路径的改成绝对路径
76+
# pattern = "(<img .*?src=\")(.*?)(\")"
77+
#
78+
# def func(m):
79+
# if not m.group(3).startswith("http"):
80+
# rtn = m.group(1) + domain + m.group(2) + m.group(3)
81+
# return rtn
82+
# else:
83+
# return m.group(1) + m.group(2) + m.group(3)
84+
#
85+
# html = re.compile(pattern).sub(func, html)
86+
html = html_template.format(content=html)
87+
html = html.encode("utf-8")
88+
with open(name, 'wb') as f:
89+
f.write(html)
90+
return name
91+
92+
except Exception as e:
93+
# logging.error("解析错误: " + e, exc_info=True)
94+
print(e)
95+
96+
97+
def save_pdf(htmls, file_name):
98+
"""
99+
把所有html文件保存到pdf文件
100+
:param htmls: html文件列表
101+
:param file_name: pdf文件名
102+
:return:
103+
"""
104+
options = {
105+
'page-size': 'Letter',
106+
'margin-top': '0.75in',
107+
'margin-right': '0.75in',
108+
'margin-bottom': '0.75in',
109+
'margin-left': '0.75in',
110+
'encoding': "UTF-8",
111+
'custom-header': [
112+
('Accept-Encoding', 'gzip')
113+
],
114+
'cookie': [
115+
('cookie-name1', 'cookie-value1'),
116+
('cookie-name2', 'cookie-value2'),
117+
],
118+
'outline-depth': 10,
119+
}
120+
pdfkit.from_file(htmls, file_name, options=options)
121+
122+
123+
def find_last(string, char):
124+
last_position = -1
125+
while True:
126+
position = string.find(char, last_position + 1)
127+
if position == -1:
128+
return last_position
129+
last_position = position
130+
131+
132+
def get_domain(url):
133+
r = urlparse(url)
134+
return r.scheme + "://" + r.netloc
135+
136+
137+
@click.command()
138+
@click.option('--url', prompt='输入要爬取的runoob教程主页地址', help='runoob网站上某一教程的主页地址')
139+
@click.option('--file', prompt='输入PDF文件的保存名称', help='不需要后缀.pdf，只需要提供名称即可')
140+
def main(url, file):
141+
start = time.time()
142+
urls = get_url_list(url)
143+
file_name = u"%s.pdf" % file
144+
htmls = [parse_url_to_html(url, str(index) + ".html") for index, url in enumerate(urls)]
145+
print(htmls)
146+
save_pdf(htmls, file_name)
147+
148+
for html in htmls:
149+
os.remove(html)
150+
151+
total_time = time.time() - start
152+
print(u"总共耗时：%f 秒" % total_time)
153+
154+
155+
if __name__ == '__main__':
156+
main()

‎runoob2pdf/runoob2pdf_1.jpg‎

24.9 KB
Loading

‎runoob2pdf/runoob2pdf_2.jpg‎

95.6 KB
Loading

‎runoob2pdf/runoob2pdf_3.jpg‎

93.5 KB
Loading

0 commit comments

Comments
 (0)