Skip to content

Commit 359c240

Browse files
liuhuanyongliuhuanyong
committed
add scrapy mode
1 parent 4844ea0 commit 359c240

3,564 files changed

Lines changed: 66615 additions & 12955 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

news/2018-04-17 14:16:18@封杀中兴！在牵强的理由背后美国打的什么算盘？.txt renamed to EventMonitor/__init__.py

File renamed without changes.

‎EventMonitor/__pycache__/__init__.cpython-36.pyc‎

145 Bytes
Binary file not shown.

‎EventMonitor/__pycache__/items.cpython-36.pyc‎

466 Bytes
Binary file not shown.

‎EventMonitor/__pycache__/pipelines.cpython-36.pyc‎

1.35 KB
Binary file not shown.

‎EventMonitor/__pycache__/settings.cpython-36.pyc‎

474 Bytes
Binary file not shown.

‎EventMonitor/crawl.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/usr/bin/env python3
2+
# coding: utf-8
3+
# File: crawl.py.py
4+
# Author: lhy<lhy_in_blcu@126.com,https://huangyong.github.io>
5+
# Date: 18-7-15
6+
from scrapy import cmdline
7+
projcet_name = 'eventspider'
8+
event_list = ['江歌被害案', '红黄蓝幼儿园虐童','于欢杀死辱母者','白银连环杀人',
9+
'章莹颖失踪','杭州保姆纵火','榆林产妇坠楼','老虎咬人事件',
10+
'魏则西事件','雷洋涉“嫖娼”致死','如家酒店女子遇袭','罗一笑事件',
11+
'徐玉玉电信诈骗身亡']
12+
13+
for event in event_list:
14+
cmdline.execute("scrapy crawl {0} -a keyword={1}".format(projcet_name, event).split())

‎EventMonitor/items.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Define here the models for your scraped items
4+
#
5+
# See documentation in:
6+
# https://doc.scrapy.org/en/latest/topics/items.html
7+
8+
import scrapy
9+
10+
11+
class EventmonitorItem(scrapy.Item):
12+
# define the fields for your item here like:
13+
# name = scrapy.Field()
14+
keyword = scrapy.Field()
15+
news_url = scrapy.Field()
16+
news_time = scrapy.Field()
17+
news_date = scrapy.Field()
18+
news_title = scrapy.Field()
19+
news_content = scrapy.Field()

‎EventMonitor/middlewares.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Define here the models for your spider middleware
4+
#
5+
# See documentation in:
6+
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
7+
8+
from scrapy import signals
9+
10+
11+
class EventmonitorSpiderMiddleware(object):
12+
# Not all methods need to be defined. If a method is not defined,
13+
# scrapy acts as if the spider middleware does not modify the
14+
# passed objects.
15+
16+
@classmethod
17+
def from_crawler(cls, crawler):
18+
# This method is used by Scrapy to create your spiders.
19+
s = cls()
20+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
21+
return s
22+
23+
def process_spider_input(self, response, spider):
24+
# Called for each response that goes through the spider
25+
# middleware and into the spider.
26+
27+
# Should return None or raise an exception.
28+
return None
29+
30+
def process_spider_output(self, response, result, spider):
31+
# Called with the results returned from the Spider, after
32+
# it has processed the response.
33+
34+
# Must return an iterable of Request, dict or Item objects.
35+
for i in result:
36+
yield i
37+
38+
def process_spider_exception(self, response, exception, spider):
39+
# Called when a spider or process_spider_input() method
40+
# (from other spider middleware) raises an exception.
41+
42+
# Should return either None or an iterable of Response, dict
43+
# or Item objects.
44+
pass
45+
46+
def process_start_requests(self, start_requests, spider):
47+
# Called with the start requests of the spider, and works
48+
# similarly to the process_spider_output() method, except
49+
# that it doesn’t have a response associated.
50+
51+
# Must return only requests (not items).
52+
for r in start_requests:
53+
yield r
54+
55+
def spider_opened(self, spider):
56+
spider.logger.info('Spider opened: %s' % spider.name)
57+
58+
59+
class EventmonitorDownloaderMiddleware(object):
60+
# Not all methods need to be defined. If a method is not defined,
61+
# scrapy acts as if the downloader middleware does not modify the
62+
# passed objects.
63+
64+
@classmethod
65+
def from_crawler(cls, crawler):
66+
# This method is used by Scrapy to create your spiders.
67+
s = cls()
68+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
69+
return s
70+
71+
def process_request(self, request, spider):
72+
# Called for each request that goes through the downloader
73+
# middleware.
74+
75+
# Must either:
76+
# - return None: continue processing this request
77+
# - or return a Response object
78+
# - or return a Request object
79+
# - or raise IgnoreRequest: process_exception() methods of
80+
# installed downloader middleware will be called
81+
return None
82+
83+
def process_response(self, request, response, spider):
84+
# Called with the response returned from the downloader.
85+
86+
# Must either;
87+
# - return a Response object
88+
# - return a Request object
89+
# - or raise IgnoreRequest
90+
return response
91+
92+
def process_exception(self, request, exception, spider):
93+
# Called when a download handler or a process_request()
94+
# (from other downloader middleware) raises an exception.
95+
96+
# Must either:
97+
# - return None: continue processing this exception
98+
# - return a Response object: stops process_exception() chain
99+
# - return a Request object: stops process_exception() chain
100+
pass
101+
102+
def spider_opened(self, spider):
103+
spider.logger.info('Spider opened: %s' % spider.name)

‎EventMonitor/pipelines.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Define your item pipelines here
4+
#
5+
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
6+
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
7+
8+
import os
9+
class EventmonitorPipeline(object):
10+
def __init__(self):
11+
CUR = '/'.join(os.path.abspath(__file__).split('/')[:-2])
12+
self.news_path = os.path.join(CUR, 'news')
13+
if not os.path.exists(self.news_path):
14+
os.makedirs(self.news_path)
15+
16+
'''处理数据流'''
17+
def process_item(self, item, spider):
18+
print(item)
19+
keyword = item['keyword']
20+
event_path = os.path.join(self.news_path, keyword)
21+
if not os.path.exists(event_path):
22+
os.makedirs(event_path)
23+
filename = os.path.join(event_path, item['news_date'] + '＠' + item['news_title'])
24+
self.save_localfile(filename, item['news_title'], item['news_time'], item['news_content'])
25+
return item
26+
27+
'''将内容保存至文件当中'''
28+
def save_localfile(self, filename, title, pubtime, content):
29+
with open(filename, 'w+') as f:
30+
f.write('标题:{0}\n'.format(title))
31+
f.write('发布时间:{0}\n'.format(pubtime))
32+
f.write('正文:{0}\n'.format(content))
33+
f.close()

‎EventMonitor/settings.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Scrapy settings for EventMonitor project
4+
#
5+
# For simplicity, this file contains only settings considered important or
6+
# commonly used. You can find more settings consulting the documentation:
7+
#
8+
# https://doc.scrapy.org/en/latest/topics/settings.html
9+
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
10+
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
11+
12+
BOT_NAME = 'EventMonitor'
13+
14+
SPIDER_MODULES = ['EventMonitor.spiders']
15+
NEWSPIDER_MODULE = 'EventMonitor.spiders'
16+
17+
18+
# Crawl responsibly by identifying yourself (and your website) on the user-agent
19+
#USER_AGENT = 'EventMonitor (+http://www.yourdomain.com)'
20+
21+
# Obey robots.txt rules
22+
ROBOTSTXT_OBEY = False
23+
24+
# Configure maximum concurrent requests performed by Scrapy (default: 16)
25+
#CONCURRENT_REQUESTS = 32
26+
27+
# Configure a delay for requests for the same website (default: 0)
28+
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
29+
# See also autothrottle settings and docs
30+
#DOWNLOAD_DELAY = 3
31+
# The download delay setting will honor only one of:
32+
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
33+
#CONCURRENT_REQUESTS_PER_IP = 16
34+
35+
# Disable cookies (enabled by default)
36+
#COOKIES_ENABLED = False
37+
38+
# Disable Telnet Console (enabled by default)
39+
#TELNETCONSOLE_ENABLED = False
40+
41+
# Override the default request headers:
42+
DEFAULT_REQUEST_HEADERS = {
43+
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
44+
'Accept-Language': 'en',
45+
}
46+
47+
# Enable or disable spider middlewares
48+
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
49+
#SPIDER_MIDDLEWARES = {
50+
# 'EventMonitor.middlewares.EventmonitorSpiderMiddleware': 543,
51+
#}
52+
53+
# Enable or disable downloader middlewares
54+
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
55+
#DOWNLOADER_MIDDLEWARES = {
56+
# 'EventMonitor.middlewares.EventmonitorDownloaderMiddleware': 543,
57+
#}
58+
59+
# Enable or disable extensions
60+
# See https://doc.scrapy.org/en/latest/topics/extensions.html
61+
#EXTENSIONS = {
62+
# 'scrapy.extensions.telnet.TelnetConsole': None,
63+
#}
64+
65+
# Configure item pipelines
66+
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
67+
ITEM_PIPELINES = {
68+
'EventMonitor.pipelines.EventmonitorPipeline': 300,
69+
}
70+
71+
# Enable and configure the AutoThrottle extension (disabled by default)
72+
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
73+
#AUTOTHROTTLE_ENABLED = True
74+
# The initial download delay
75+
#AUTOTHROTTLE_START_DELAY = 5
76+
# The maximum download delay to be set in case of high latencies
77+
#AUTOTHROTTLE_MAX_DELAY = 60
78+
# The average number of requests Scrapy should be sending in parallel to
79+
# each remote server
80+
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
81+
# Enable showing throttling stats for every response received:
82+
#AUTOTHROTTLE_DEBUG = False
83+
84+
# Enable and configure HTTP caching (disabled by default)
85+
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
86+
#HTTPCACHE_ENABLED = True
87+
#HTTPCACHE_EXPIRATION_SECS = 0
88+
#HTTPCACHE_DIR = 'httpcache'
89+
#HTTPCACHE_IGNORE_HTTP_CODES = []
90+
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

0 commit comments

Comments
 (0)