add robots.txt

Boris-code · Boris-code · commit ecbd35c301e6 · 2021-11-10T23:08:50.000+08:00
diff --git a/docs/index.html b/docs/index.html
@@ -5,14 +5,16 @@
     <meta charset="UTF-8">
     <title>feapder-document</title>
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
-    <meta name="description" content="feapder是一款支持分布式、批次采集、任务防丢、报警丰富的python爬虫框架">
+    <meta name="description" content="feapder官方文档">
     <meta name="keywords"
-        content="feapder,scrapy,feapder文档,scrapy文档,feapder document,scrapy document,feapder 教程,scrapy 教程,爬虫,爬虫框架,Boris,批次爬虫,分布式爬虫">
+        content="feapder官方文档,feapder文档,feapder document,feapder 教程,爬虫框架,Boris,批次爬虫,分布式爬虫">
     <meta name="viewport"
         content="width=device-width, user-scalable=no, initial-scale=1.0, maximum-scale=1.0, minimum-scale=1.0">
     <meta name="baidu-site-verification" content="code-5vXrzaZgX6" />
     <link rel="stylesheet" href="./lib/docsify/lib/themes/vue.css">
 
+    <meta name="robots" content="all"/>
+
     <!-- 谷歌统计 -->
     <!-- Global site tag (gtag.js) - Google Analytics -->
     <script async src="https://www.googletagmanager.com/gtag/js?id=G-KS7S55K3YN"></script>
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1 @@
+User-agent: *
