Skip to content

Commit 8af54ff

Browse files
jaggerwangjaggerwang
committed
restructure seiya
1 parent 15213d8 commit 8af54ff

25 files changed

Lines changed: 552 additions & 0 deletions

‎seiya/__init__.py‎

Whitespace-only changes.

‎seiya/analysis/__init__.py‎

Whitespace-only changes.

‎seiya/models.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
from sqlalchemy import create_engine, Column, String, Integer
2+
from sqlalchemy.ext.declarative import declarative_base
3+
4+
engine = create_engine(
5+
'mysql://root@localhost:3306/data-analysis-project?charset=utf8')
6+
Base = declarative_base()
7+
8+
9+
class JobModel(Base):
10+
__tablename__ = 'job'
11+
12+
id = Column(Integer, primary_key=True)
13+
title = Column(String(64))
14+
city = Column(String(16))
15+
salary_lower = Column(Integer)
16+
salary_upper = Column(Integer)
17+
experience_lower = Column(Integer)
18+
experience_upper = Column(Integer)
19+
education = Column(String(16))
20+
tags = Column(String(256))
21+
company = Column(String(32))

‎seiya/scrapy.cfg‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Automatically created by: scrapy startproject
2+
#
3+
# For more information about the [deploy] section see:
4+
# https://scrapyd.readthedocs.io/en/latest/deploy.html
5+
6+
[settings]
7+
default = spider.settings
8+
9+
[deploy]
10+
#url = http://localhost:6800/
11+
project = spider

‎seiya/spider/__init__.py‎

Whitespace-only changes.

‎seiya/spider/items.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import scrapy
2+
3+
4+
class JobItem(scrapy.Item):
5+
title = scrapy.Field()
6+
city = scrapy.Field()
7+
salary = scrapy.Field()
8+
experience = scrapy.Field()
9+
education = scrapy.Field()
10+
tags = scrapy.Field()
11+
company = scrapy.Field()

‎seiya/spider/middlewares.py‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Define here the models for your spider middleware
4+
#
5+
# See documentation in:
6+
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
7+
8+
from scrapy import signals
9+
10+
11+
class SpiderSpiderMiddleware(object):
12+
# Not all methods need to be defined. If a method is not defined,
13+
# scrapy acts as if the spider middleware does not modify the
14+
# passed objects.
15+
16+
@classmethod
17+
def from_crawler(cls, crawler):
18+
# This method is used by Scrapy to create your spiders.
19+
s = cls()
20+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
21+
return s
22+
23+
def process_spider_input(self, response, spider):
24+
# Called for each response that goes through the spider
25+
# middleware and into the spider.
26+
27+
# Should return None or raise an exception.
28+
return None
29+
30+
def process_spider_output(self, response, result, spider):
31+
# Called with the results returned from the Spider, after
32+
# it has processed the response.
33+
34+
# Must return an iterable of Request, dict or Item objects.
35+
for i in result:
36+
yield i
37+
38+
def process_spider_exception(self, response, exception, spider):
39+
# Called when a spider or process_spider_input() method
40+
# (from other spider middleware) raises an exception.
41+
42+
# Should return either None or an iterable of Response, dict
43+
# or Item objects.
44+
pass
45+
46+
def process_start_requests(self, start_requests, spider):
47+
# Called with the start requests of the spider, and works
48+
# similarly to the process_spider_output() method, except
49+
# that it doesn’t have a response associated.
50+
51+
# Must return only requests (not items).
52+
for r in start_requests:
53+
yield r
54+
55+
def spider_opened(self, spider):
56+
spider.logger.info('Spider opened: %s' % spider.name)
57+
58+
59+
class SpiderDownloaderMiddleware(object):
60+
# Not all methods need to be defined. If a method is not defined,
61+
# scrapy acts as if the downloader middleware does not modify the
62+
# passed objects.
63+
64+
@classmethod
65+
def from_crawler(cls, crawler):
66+
# This method is used by Scrapy to create your spiders.
67+
s = cls()
68+
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
69+
return s
70+
71+
def process_request(self, request, spider):
72+
# Called for each request that goes through the downloader
73+
# middleware.
74+
75+
# Must either:
76+
# - return None: continue processing this request
77+
# - or return a Response object
78+
# - or return a Request object
79+
# - or raise IgnoreRequest: process_exception() methods of
80+
# installed downloader middleware will be called
81+
return None
82+
83+
def process_response(self, request, response, spider):
84+
# Called with the response returned from the downloader.
85+
86+
# Must either;
87+
# - return a Response object
88+
# - return a Request object
89+
# - or raise IgnoreRequest
90+
return response
91+
92+
def process_exception(self, request, exception, spider):
93+
# Called when a download handler or a process_request()
94+
# (from other downloader middleware) raises an exception.
95+
96+
# Must either:
97+
# - return None: continue processing this exception
98+
# - return a Response object: stops process_exception() chain
99+
# - return a Request object: stops process_exception() chain
100+
pass
101+
102+
def spider_opened(self, spider):
103+
spider.logger.info('Spider opened: %s' % spider.name)

‎seiya/spider/pipelines.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
import re
2+
3+
from sqlalchemy.orm import sessionmaker
4+
5+
from seiya.models import engine, JobModel
6+
from seiya.spider.items import JobItem
7+
8+
9+
class PersistentPipeline(object):
10+
def open_spider(self, spider):
11+
self.session = sessionmaker(bind=engine)()
12+
13+
def close_spider(self, spider):
14+
self.session.commit()
15+
self.session.close()
16+
17+
def process_item(self, item, spider):
18+
if isinstance(item, JobItem):
19+
return self._process_job_item(item)
20+
else:
21+
return item
22+
23+
def _process_job_item(self, item):
24+
city = item['city'].split('·')[0]
25+
26+
salary_lower, salary_upper = 0, 0
27+
m = re.match(r'[^\d]*(\d+)k-(\d+)k', item['salary'])
28+
if m is not None:
29+
salary_lower, salary_upper = int(m.group(1)), int(m.group(2))
30+
31+
experience_lower, experience_upper = 0, 0
32+
m = re.match(r'[^\d]*(\d+)-(\d+)', item['experience'])
33+
if m is not None:
34+
experience_lower, experience_upper = int(
35+
m.group(1)), int(m.group(2))
36+
37+
tags = ' '.join(item['tags'])
38+
39+
model = JobModel(
40+
title=item['title'],
41+
city=city,
42+
salary_lower=salary_lower,
43+
salary_upper=salary_upper,
44+
experience_lower=experience_lower,
45+
experience_upper=experience_upper,
46+
education=item['education'],
47+
tags=tags,
48+
company=item['company'],
49+
)
50+
51+
self.session.add(model)
52+
53+
return item

‎seiya/spider/settings.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# -*- coding: utf-8 -*-
2+
3+
# Scrapy settings for spider project
4+
#
5+
# For simplicity, this file contains only settings considered important or
6+
# commonly used. You can find more settings consulting the documentation:
7+
#
8+
# https://doc.scrapy.org/en/latest/topics/settings.html
9+
# https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
10+
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html
11+
12+
BOT_NAME = 'spider'
13+
14+
SPIDER_MODULES = ['spider.spiders']
15+
NEWSPIDER_MODULE = 'spider.spiders'
16+
17+
18+
# Crawl responsibly by identifying yourself (and your website) on the user-agent
19+
#USER_AGENT = 'spider (+http://www.yourdomain.com)'
20+
21+
# Obey robots.txt rules
22+
ROBOTSTXT_OBEY = True
23+
24+
# Configure maximum concurrent requests performed by Scrapy (default: 16)
25+
#CONCURRENT_REQUESTS = 32
26+
27+
# Configure a delay for requests for the same website (default: 0)
28+
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
29+
# See also autothrottle settings and docs
30+
DOWNLOAD_DELAY = 30
31+
# The download delay setting will honor only one of:
32+
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
33+
#CONCURRENT_REQUESTS_PER_IP = 16
34+
35+
# Disable cookies (enabled by default)
36+
# COOKIES_ENABLED = True
37+
38+
# Disable Telnet Console (enabled by default)
39+
#TELNETCONSOLE_ENABLED = False
40+
41+
# Override the default request headers:
42+
# DEFAULT_REQUEST_HEADERS = {
43+
# 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
44+
# 'Accept-Language': 'en',
45+
# }
46+
47+
# Enable or disable spider middlewares
48+
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
49+
# SPIDER_MIDDLEWARES = {
50+
# 'spider.middlewares.SpiderSpiderMiddleware': 543,
51+
# }
52+
53+
# Enable or disable downloader middlewares
54+
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
55+
# DOWNLOADER_MIDDLEWARES = {
56+
# 'spider.middlewares.SpiderDownloaderMiddleware': 543,
57+
# }
58+
59+
# Enable or disable extensions
60+
# See https://doc.scrapy.org/en/latest/topics/extensions.html
61+
# EXTENSIONS = {
62+
# 'scrapy.extensions.telnet.TelnetConsole': None,
63+
# }
64+
65+
# Configure item pipelines
66+
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
67+
ITEM_PIPELINES = {
68+
'spider.pipelines.PersistentPipeline': 300,
69+
}
70+
71+
# Enable and configure the AutoThrottle extension (disabled by default)
72+
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
73+
#AUTOTHROTTLE_ENABLED = True
74+
# The initial download delay
75+
#AUTOTHROTTLE_START_DELAY = 5
76+
# The maximum download delay to be set in case of high latencies
77+
#AUTOTHROTTLE_MAX_DELAY = 60
78+
# The average number of requests Scrapy should be sending in parallel to
79+
# each remote server
80+
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
81+
# Enable showing throttling stats for every response received:
82+
#AUTOTHROTTLE_DEBUG = False
83+
84+
# Enable and configure HTTP caching (disabled by default)
85+
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
86+
#HTTPCACHE_ENABLED = True
87+
#HTTPCACHE_EXPIRATION_SECS = 0
88+
#HTTPCACHE_DIR = 'httpcache'
89+
#HTTPCACHE_IGNORE_HTTP_CODES = []
90+
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

‎seiya/spider/spiders/__init__.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# This package will contain the spiders of your Scrapy project
2+
#
3+
# Please refer to the documentation for information on how to create and manage
4+
# your spiders.

0 commit comments

Comments
 (0)