add ytb comment

lzjun · lzjun · commit e442c2549448 · 2019-12-17T20:17:18.000+08:00
diff --git a/Pipfile b/Pipfile
@@ -0,0 +1,12 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+
+[packages]
+requests = {extras = ["socks"],version = "*"}
+
+[requires]
+python_version = "3.7"
diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/blog/crawler_blog_async.py b/blog/crawler_blog_async.py
@@ -15,7 +15,7 @@
 
 headers = {
     'Connection': 'Keep-Alive',
-    'Accept': 'text/html, application/xhtml+xml, */*',
+    'Accept': 'text.txt/html, application/xhtml+xml, */*',
     'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Sa",
     "Referer": "http://www.jobbole.com/",
diff --git a/heart2/heart.py b/heart2/heart.py
@@ -52,7 +52,7 @@ def fetch_data(uid=None, container_id=None):
         for card in cards:
             # 每条微博的正文内容
             if card.get("card_type") == 9:
-                text = card.get("mblog").get("text")
+                text = card.get("mblog").get("text.txt")
                 text = clean_html(text)
                 blogs.append(text)
         page += 1
diff --git a/juejin/juejin.py b/juejin/juejin.py
@@ -15,7 +15,7 @@
 
 headers = {
     'Connection': 'Keep-Alive',
-    'Accept': 'text/html, application/xhtml+xml, */*',
+    'Accept': 'text.txt/html, application/xhtml+xml, */*',
     'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
     "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
     "HOST": "api.leancloud.cn",
diff --git a/juejin/test.py b/juejin/test.py
diff --git a/xingqiu/xingqiu.py b/xingqiu/xingqiu.py
@@ -33,7 +33,7 @@ def str_to_dict(s, join_symbol="\n", split_symbol=":"):
 
 # 直接从浏览器里面拷贝过来的，请替换成你自己的浏览器中的内容
 headers = """
-Accept: application/json, text/plain, */*
+Accept: application/json, text.txt/plain, */*
 Accept-Encoding: gzip, deflate, br
 Accept-Language: zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7
 Authorization: 8EAE71AC-081C-FE82-BE8C-954696
@@ -56,7 +56,7 @@ def crawl(url):
     if len(topics) <= 1:
         return
     for i in topics:
-        print(i.get("talk").get("text")[:10])
+        print(i.get("talk").get("text.txt")[:10])
         db.topics.insert_one(i)
     else:
         last_time = i.get("create_time")
diff --git a/youbute_comment/__init__.py b/youbute_comment/__init__.py
@@ -0,0 +1,60 @@
+"""
+获取youtube视频下的评论
+
+思路：
+
+基于youtube官方的API来获取, 这里是关于如何初始化配置的文档https://developers.google.com/youtube/v3/getting-started
+
+接口文档：https://developers.google.com/youtube/v3/docs/channelSections/list
+
+视频地址：https://www.youtube.com/watch?v=FWMIPukvdsQ
+
+"""
+import requests
+
+#  在 API Console 配置生成
+key = "xxxxxx"
+# 视频ID
+videoId = "FWMIPukvdsQ"
+
+url = f"https://www.googleapis.com/youtube/v3/commentThreads?" \
+      f"key={key}&" \
+      f"textFormat=plainText&" \
+      f"part=snippet&" \
+      f"videoId={videoId}&" \
+      f"maxResults=100"
+
+proxies = {
+    'http': 'socks5://127.0.0.1:1080',
+    'https': 'socks5://127.0.0.1:1080',
+}
+
+
+# 获取一下页的凭证
+
+
+def main():
+    nextPageToken = "QURTSl9pMkR0YkxlcE1iOHhlLU1lNi1XWGhzTHdpUmlCN2w1UmJDNlVBaEhnT1dyejVFb3dnVGdWbExRSFNtMVRrNjE1TWVPWC04UVN2VGJrMkhjZ01KVmJpNllrRlVkdUFRWk1yVHp2cW1ZbjVNcXpFc2ZzRlI3ZkRlM3ZPUm1CalZSX1NkaE9qcEY4Tl8yUWRyMmN3"
+    while nextPageToken is not None:
+        if nextPageToken:
+            params = {"pageToken": nextPageToken}
+        else:
+            params = None
+        res = requests.get(url, proxies=proxies, params=params)
+        data = res.json()
+        import pprint
+        nextPageToken = data.get("nextPageToken")
+        print(nextPageToken)
+        items = data.get("items")
+        for item in items:
+            comment = item.get("snippet").get("topLevelComment").get("snippet").get("textDisplay")
+            print(comment)
+        import time
+        time.sleep(1)
+        print("==================")
+
+
+
+
+if __name__ == '__main__':
+    main()
diff --git a/youbute_comment/text.txt b/youbute_comment/text.txt
