爬虫模块bug修改

houko · houko · commit a8aba0af3073 · 2017-09-08T09:56:39.000+08:00
diff --git a/README.md b/README.md
@@ -45,38 +45,23 @@
 
 
 ### 更新日志
+由于更新日志占用篇幅太多，单独用一个文件存放。链接地址 [更新日志(详细)](/changeLog.md),这里只展示更新简介
+
 - 2017-09-02 api模块: 添加swagger-bootstrap-ui,和原有ui并行存在。       
 http://localhost:8080 默认UI           
-http://localhost:808/doc.html bootstrap-ui   
-
-```
-1. spring boot版本从1.4.3更新到1.5.6   
-2. 修复不配置数据库信息无法启动的bug   
-3. 版本号更新到2017.1   
-4. api模块(swagger)添加开源库swagger-bootstrap-ui，和swagger默认UI同时存在。  
-5. web模块添加数据库sql文件,导入后一键启动可直接访问到web界面。  
-```
-
-- 2017-09-06 更新记录
-
-```
-1. mybatis模块:添加USER.sql,启动后访问:http://localhost:8080 即可看到接口数据
-2. 所有模块： 添加 characterEncoding=utf8&useSSL=true 解决高版本mysql的sll警告
-3. 添加代码贡献者列表和支持者，赞助商链接。
-```
-
-![support](screenshot/support.png)
-
-#### swagger            
-![默认](screenshot/swagger-ui.png)
-
+http://localhost:808/doc.html bootstrap-ui  
 
-#### bootstrap-ui
-![bootstrap](screenshot/bootstrap.png)
+- 2017-09-02 spring boot版本从1.4.3更新到1.5.6   
+- 2017-09-02 修复不配置数据库信息无法启动的bug   
+- 2017-09-02 版本号更新到2017.1   
+- 2017-09-02 api模块(swagger)添加开源库swagger-bootstrap-ui，和swagger默认UI同时存在。  
+- 2017-09-02 web模块添加数据库sql文件,导入后一键启动可直接访问到web界面。  
+- 2017-09-06 mybatis模块:添加USER.sql,启动后访问:http://localhost:8080 即可看到接口数据
+- 2017-09-06 所有模块： 添加 characterEncoding=utf8&useSSL=true 解决高版本mysql的sll警告
+- 2017-09-06 添加代码贡献者列表和支持者，赞助商链接。
+- 2017-09-08 crawler模块(网络爬虫):修复本地文件目录不存在会报错的bug。处理方式为：不存在则自动创建
 
-![bootstrap](screenshot/interface.png)
 
-![bootstrap](screenshot/api.png)
 
 
 ###  项目说明
diff --git a/changeLog.md b/changeLog.md
@@ -0,0 +1,45 @@
+- 2017-09-02 api模块: 添加swagger-bootstrap-ui,和原有ui并行存在。       
+http://localhost:8080 默认UI           
+http://localhost:808/doc.html bootstrap-ui   
+
+```
+1. spring boot版本从1.4.3更新到1.5.6   
+2. 修复不配置数据库信息无法启动的bug   
+3. 版本号更新到2017.1   
+4. api模块(swagger)添加开源库swagger-bootstrap-ui，和swagger默认UI同时存在。  
+5. web模块添加数据库sql文件,导入后一键启动可直接访问到web界面。  
+```
+
+
+
+![support](screenshot/support.png)
+
+#### swagger            
+![默认](screenshot/swagger-ui.png)
+
+
+#### bootstrap-ui
+![bootstrap](screenshot/bootstrap.png)
+
+![bootstrap](screenshot/interface.png)
+
+![bootstrap](screenshot/api.png)
+
+
+
+- 2017-09-06 更新记录
+
+```
+1. mybatis模块:添加USER.sql,启动后访问:http://localhost:8080 即可看到接口数据
+2. 所有模块： 添加 characterEncoding=utf8&useSSL=true 解决高版本mysql的sll警告
+3. 添加代码贡献者列表和支持者，赞助商链接。
+```
+
+- 2017-09-08 更新记录
+
+```
+1. crawler模块(网络爬虫):修复本地文件目录不存在会报错的bug。处理方式为：不存在则自动创建
+```
+
+![crawler](screenshot/crawler.png)
+![yys](screenshot/yys.png)
diff --git a/core/src/main/java/info/xiaomo/core/untils/DownUtil.java b/core/src/main/java/info/xiaomo/core/untils/DownUtil.java
@@ -1,5 +1,7 @@
 package info.xiaomo.core.untils;
 
+import lombok.extern.slf4j.Slf4j;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.InputStream;
@@ -9,8 +11,10 @@
 
 /**
  * author 小莫 (https://xiaomo.info) (https://github.com/xiaomoinfo)
+ *
  * @created : 2016/12/26 13:25
  */
+@Slf4j
 public class DownUtil {
 
     public static void download(String urlString) throws Exception {
@@ -40,10 +44,11 @@ public static void download(String urlString) throws Exception {
 
     /**
      * 下载图片
+     *
      * @param urlString url
-     * @param filePath 存储路径  D:\MIR\config\data\
+     * @param filePath  存储路径  D:\MIR\config\data\
      */
-    public static void download(String urlString,String filePath) throws Exception {
+    public static void download(String urlString, String filePath) throws Exception {
         File file = new File(urlString);
         String filename = file.getName();
         // 构造URL
@@ -57,7 +62,16 @@ public static void download(String urlString,String filePath) throws Exception {
         // 读取到的数据长度
         int len;
         // 输出的文件流
-        OutputStream os = new FileOutputStream(filePath+filename);
+
+        File output = new File(filePath);
+        if (!output.exists()){
+            boolean res = output.mkdir();
+            if (res){
+               log.debug("{} 目录创建成功", filePath);
+            }
+        }
+
+        OutputStream os = new FileOutputStream(filePath + filename);
         // 开始读取
         while ((len = is.read(bs)) != -1) {
             os.write(bs, 0, len);
diff --git a/crawler/src/main/java/info/xiaomo/crawler/CrawlerMain.java b/crawler/src/main/java/info/xiaomo/crawler/CrawlerMain.java
@@ -5,25 +5,27 @@
 import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.Configuration;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
 import org.springframework.scheduling.annotation.EnableScheduling;
 
 /**
  * 把今天最好的表现当作明天最新的起点．．～
  * いま 最高の表現 として 明日最新の始発．．～
  * Today the best performance  as tomorrow newest starter!
  * Created by IntelliJ IDEA.
- *
+ * <p>
  * author: xiaomo
  * github: https://github.com/xiaomoinfo
  * email: xiaomo@xiaomo.info
-
+ * <p>
  * Date: 2016/4/1 15:38
  * Copyright(©) 2015 by xiaomo.
  **/
 @Configuration
 @EnableAutoConfiguration
 @EnableScheduling
 @Configurable
+@EnableJpaRepositories("info.xiaomo.*.dao")
 @ComponentScan("info.xiaomo")
 public class CrawlerMain {
     public static void main(String[] args) throws Exception {
diff --git a/crawler/src/main/java/info/xiaomo/crawler/dao/ShikigamaDao.java b/crawler/src/main/java/info/xiaomo/crawler/dao/ShikigamaDao.java
@@ -2,11 +2,13 @@
 
 import info.xiaomo.crawler.model.ShikigamiModel;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.stereotype.Repository;
 
 /**
  * author 小莫 (https://xiaomo.info) (https://github.com/xiaomoinfo)
  * @created : 2016/12/24 15:55
  */
+@Repository
 public interface ShikigamaDao extends JpaRepository<ShikigamiModel, Long> {
 
     ShikigamiModel findByName(String name);
diff --git a/crawler/src/main/java/info/xiaomo/crawler/schedule/ScheduledTasks.java b/crawler/src/main/java/info/xiaomo/crawler/schedule/ScheduledTasks.java
@@ -52,7 +52,7 @@ public void downImage() throws Exception {
         List<ShikigamiModel> shikigamiModel = shikigamaService.findAll();
         for (ShikigamiModel aShikigamiModel : shikigamiModel) {
             String url = aShikigamiModel.getImage();
-            DownUtil.download(url, "C:\\Users\\xiaomo\\Desktop\\yss\\");
+            DownUtil.download(url, "D:\\yys\\");
             LOGGER.debug("开始下载图片:{}", url);
         }
     }
diff --git a/crawler/src/main/resources/config/application.properties b/crawler/src/main/resources/config/application.properties
@@ -0,0 +1,19 @@
+logging.config=classpath:config/logback-dev.xml
+server.port=8080
+server.session.timeout=1800
+server.max-http-header-size=20971520
+
+#datasource
+spring.datasource.url=jdbc:mysql://ip/db?characterEncoding=utf8&useSSL=true
+#  ?useUnicode=true&amp;characterEncoding=UTF-8
+spring.datasource.username=username
+spring.datasource.password=password
+spring.datasource.driver-class-name=com.mysql.jdbc.Driver
+# \u914D\u7F6E\u8FD9\u4E2A\u503C\u5C31\u53EF\u4EE5\u683C\u5F0F\u5316\u65F6\u95F4
+#spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
+#spring.jackson.time-zone=GMT+8
+
+#jpa
+spring.jpa.database-platform=org.hibernate.dialect.MySQL5Dialect
+spring.jpa.hibernate.ddl-auto=update
+spring.jpa.show-sql=true
diff --git a/screenshot/crawler.png b/screenshot/crawler.png
diff --git a/screenshot/yys.png b/screenshot/yys.png

Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ public void downImage() throws Exception {`
`52`	`52`	`List<ShikigamiModel> shikigamiModel = shikigamaService.findAll();`
`53`	`53`	`for (ShikigamiModel aShikigamiModel : shikigamiModel) {`
`54`	`54`	`String url = aShikigamiModel.getImage();`
`55`		`- DownUtil.download(url, "C:\\Users\\xiaomo\\Desktop\\yss\\");`
	`55`	`+ DownUtil.download(url, "D:\\yys\\");`
`56`	`56`	`LOGGER.debug("开始下载图片:{}", url);`
`57`	`57`	`}`
`58`	`58`	`}`