22import os
33import re
44import time
5-
5+ import logging
66import pdfkit
77import requests
88from bs4 import BeautifulSoup
@@ -30,7 +30,7 @@ def parse_url_to_html(url, name):
3030 """
3131 try :
3232 response = requests .get (url )
33- soup = BeautifulSoup (response .content , "html5lib" )
33+ soup = BeautifulSoup (response .content , 'html.parser' )
3434 # 正文
3535 body = soup .find_all (class_ = "x-wiki-content" )[0 ]
3636 # 标题
@@ -47,17 +47,22 @@ def parse_url_to_html(url, name):
4747 pattern = "(<img .*?src=\" )(.*?)(\" )"
4848
4949 def func (m ):
50- rtn = m .group (1 ) + "http://www.liaoxuefeng.com" + m .group (2 ) + m .group (3 )
51- return rtn
50+ if not m .group (3 ).startswith ("http" ):
51+ rtn = m .group (1 ) + "http://www.liaoxuefeng.com" + m .group (2 ) + m .group (3 )
52+ return rtn
53+ else :
54+ return m .group (1 )+ m .group (2 )+ m .group (3 )
5255
5356 html = re .compile (pattern ).sub (func , html )
5457 html = html_template .format (content = html )
58+ html = html .encode ("utf-8" )
5559 with open (name , 'wb' ) as f :
5660 f .write (html )
5761 return name
5862
5963 except Exception as e :
60- print (e .message )
64+
65+ logging .error ("解析错误" , exc_info = True )
6166
6267
6368def get_url_list ():
@@ -66,7 +71,7 @@ def get_url_list():
6671 :return:
6772 """
6873 response = requests .get ("http://www.liaoxuefeng.com/wiki/0014316089557264a6b348958f449949df42a6d3a2e542c000" )
69- soup = BeautifulSoup (response .content , "html5lib " )
74+ soup = BeautifulSoup (response .content , "html.parser " )
7075 menu_tag = soup .find_all (class_ = "uk-nav uk-nav-side" )[1 ]
7176 urls = []
7277 for li in menu_tag .find_all ("li" ):
0 commit comments