import requests, os

from lxml import etree

from fake_useragent import UserAgent

import time

class goblin(object):

def __init__(self):

self.url = "http://www.cbaigui.com/?paged={}"

ua = UserAgent(verify_ssl=False)

for i in range(1, 50):

self.headers = {

'User-Agent': ua.random,

'Host': 'www.cbaigui.com',

'Referer': 'http: // www.cbaigui.com /?paged={}'

}

os.mkdir("妖怪") # 创建妖怪这个文件夹 记住只有第一次运行加上，如果多次运行请屏蔽掉本行

'''发送请求 获取响应'''

def get_page(self, url):

res = requests.get(url=url, headers=self.headers)

html = res.content.decode("utf-8")

return html

'''解析数据'''

#

def parse_page(self, html):

parse_html = etree.HTML(html)

t = parse_html.xpath('//div[@class="post-inner post-hover"]/h2')

for i in t:

goblin_herf = i.xpath('./a/@href')[0].strip() # 二级页面链接

name = i.xpath('.//a/text()')[0].strip() # 对应文件夹的名字

os.mkdir("././妖怪/{}".format(name)) # 拼接文件夹

print(name, goblin_herf)

html2 = self.get_page(goblin_herf) # 第二个发生请求

parse_html2 = etree.HTML(html2)

r = parse_html2.xpath('//div[@class="entry"]/p/text()')

for rte in r:

# print(rte)w

try:

with open("./妖怪/{}/{}.txt".format(name, name), "a", encoding='utf-8') as f:

f.write(rte) # 把内容存储到对应名字的文件里

except OSError:

pass

continue

def main(self):

startPage = int(input("起始页:"))

endPage = int(input("终止页:"))

for page in range(startPage, endPage + 1):

url = self.url.format(page)

print(url)

html = self.get_page(url)

self.parse_page(html)

time.sleep(1)

print("======================第%s页爬取成功！！！！=======================" % page)

if __name__ == '__main__':

imageSpider = goblin()

imageSpider.main()