From b115b1f392d357ea4699391b8618569e324a800e Mon Sep 17 00:00:00 2001
From: "U-TQPN6VIPCMWKPWG\\Administrator" <5linhua5@gmail.com>
Date: Wed, 13 Jan 2016 14:45:27 +0800
Subject: [PATCH 0001/1225] [youku] fix extractor_proxy option

---
 src/you_get/extractors/youku.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 8202dd1dbf..38ceeb0c4a 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -129,7 +129,15 @@ def prepare(self, **kwargs):
         ssl_context = request.HTTPSHandler(
             context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
         cookie_handler = request.HTTPCookieProcessor()
-        opener = request.build_opener(ssl_context, cookie_handler)
+        if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+            proxy = parse_host(kwargs['extractor_proxy'])
+            proxy_handler = request.ProxyHandler({
+                'http': '%s:%s' % proxy,
+                'https': '%s:%s' % proxy,
+            })
+        else:
+            proxy_handler = request.ProxyHandler({})
+        opener = request.build_opener(ssl_context, cookie_handler, proxy_handler)
         opener.addheaders = [('Cookie','__ysuid={}'.format(time.time()))]
         request.install_opener(opener)
 
@@ -267,6 +275,7 @@ def extract(self, **kwargs):
                         fileid    = fileid,
                         q         = q
                     )
+                    # unset_proxy()  also strips cookies,because k.youku.com doesn't need cookies('r') for now
                     ksegs += [i['server'] for i in json.loads(get_content(u))]
             except error.HTTPError as e:
                 # Use fallback stream data in case of HTTP 404

From 56f11ab79959eed5f63d4cd752326b4d5d152ec4 Mon Sep 17 00:00:00 2001
From: xyuanmu <xyuanmu@gmail.com>
Date: Wed, 20 Jan 2016 11:29:15 +0800
Subject: [PATCH 0002/1225] Update iqiyi key

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index af8e0df0e7..2700627d6b 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -45,7 +45,7 @@
 
 '''
 def mix(tvid):
-    salt = 'd7184ccc20a84a9d8be798087386b6b8'
+    salt = '6ab6d0280511493ba85594779759d4ed'
     tm = str(randint(2000,4000))
     sc = hashlib.new('md5', bytes(salt + tm + tvid, 'utf-8')).hexdigest()
     return tm, sc, 'eknas'

From 511afb386202bd5804aa9f8dfb1cc043d3d8832d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Jan 2016 04:35:40 +0100
Subject: [PATCH 0003/1225] [vimeo] fix #879

---
 src/you_get/extractors/vimeo.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index acb0e0df57..27c05b48f1 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -59,6 +59,9 @@ def vimeo_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vimeo_download_by_channel(url, output_dir, merge, info_only)
     else:
         id = r1(r'https?://[\w.]*vimeo.com[/\w]*/(\d+)', url)
+        if id is None:
+            video_page = get_content(url, headers=fake_headers)
+            id = r1(r'"clip_id":(\d+)', video_page)
         assert id
 
         vimeo_download_by_id(id, None, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)

From 1f2c0bde94f9b78907952c59453694785153a736 Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Fri, 22 Jan 2016 17:58:35 +0800
Subject: [PATCH 0004/1225] Revert "fix zhanqitv"

This reverts commit eaab0ce3ed3054db0a12bad5e8511d5e7791f5c8.

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/zhanqi.py | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index 7d6b75b67c..a648bc4af9 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -4,10 +4,6 @@
 
 from ..common import *
 import re
-import base64
-import json
-import time
-import hashlib
 
 def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_content(url)
@@ -24,28 +20,12 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
     rtmp_base = "http://wshdl.load.cdn.zhanqi.tv/zqlive"
     vod_base = "http://dlvod.cdn.zhanqi.tv"
     rtmp_real_base = "rtmp://dlrtmp.cdn.zhanqi.tv/zqlive/"
-    room_info = "http://www.zhanqi.tv/api/static/live.roomid/"
-    KEY_MASK = "#{&..?!("
-    ak2_pattern = r'ak2":"\d-([^|]+)'
     
     if video_type == "LIVE":
         rtmp_id = match1(html, rtmp_id_patt).replace('\\/','/')
         #request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
         #real_url = get_html(request_url)
-        html2 = get_content(room_info + rtmp_id.split("_")[0] + ".json")
-        json_data = json.loads(html2)
-        cdns = json_data["data"]["flashvars"]["cdns"]
-        cdns = base64.b64decode(cdns).decode("utf-8")
-        cdn = match1(cdns, ak2_pattern)
-        cdn = base64.b64decode(cdn).decode("utf-8")
-        key = ''
-        i = 0
-        while(i < len(cdn)):
-            key = key + chr(ord(cdn[i]) ^ ord(KEY_MASK[i % 8]))
-            i = i + 1
-        time_hex = hex(int(time.time()))[2:]
-        key = hashlib.md5(bytes(key + "/zqlive/" + rtmp_id + time_hex, "utf-8")).hexdigest()
-        real_url = rtmp_real_base + '/' + rtmp_id + "?k=" + key + "&t=" + time_hex
+        real_url = rtmp_real_base + '/' + rtmp_id
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
             download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)

From 75d8367ade90814a0da3414df5a80c42a2803237 Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Fri, 22 Jan 2016 17:59:09 +0800
Subject: [PATCH 0005/1225] Revert "[ZhanQi]temporary fix using RTMP url"

This reverts commit 953c2547bcd60204895974f85ba28a5898443ee6.

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/zhanqi.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index a648bc4af9..f450f9549e 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -19,17 +19,15 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
     title = unescape_html(title)
     rtmp_base = "http://wshdl.load.cdn.zhanqi.tv/zqlive"
     vod_base = "http://dlvod.cdn.zhanqi.tv"
-    rtmp_real_base = "rtmp://dlrtmp.cdn.zhanqi.tv/zqlive/"
     
     if video_type == "LIVE":
         rtmp_id = match1(html, rtmp_id_patt).replace('\\/','/')
-        #request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
-        #real_url = get_html(request_url)
-        real_url = rtmp_real_base + '/' + rtmp_id
+        request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
+        real_url = get_html(request_url)
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
-            download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
-            #download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+            #download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
+            download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
     elif video_type == "VOD":
         vod_m3u8_request = vod_base + match1(html, vod_m3u8_id_patt).replace('\\/','/')
         vod_m3u8 = get_html(vod_m3u8_request)
@@ -49,7 +47,6 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
             download_urls(real_url, title, type_ or 'ts', size, output_dir, merge = merge)
     else:
         NotImplementedError('Unknown_video_type')
-
 site_info = "zhanqi.tv"
 download = zhanqi_download
 download_playlist = playlist_not_supported('zhanqi')

From 46f8bc18ea5b4c59c6844d8cab343176fea98e83 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 23 Jan 2016 11:38:11 +0800
Subject: [PATCH 0006/1225] [qq] support for embedded video URLs

e.g. http://v.qq.com/iframe/player.html?vid=q01536xb5rb&tiny=0&auto=0
---
 src/you_get/extractors/qq.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index b1d8eb0fa3..01d4311f0b 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -16,9 +16,14 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    content = get_html(url)
-    vid = match1(content, r'vid\s*:\s*"\s*([^"]+)"')
-    title = match1(content, r'title\s*:\s*"\s*([^"]+)"')
+    if 'iframe/player.html' in url:
+        vid = match1(url, r'\bvid=(\w+)')
+        # for embedded URLs; don't know what the title is
+        title = vid
+    else:
+        content = get_html(url)
+        vid = match1(content, r'vid\s*:\s*"\s*([^"]+)"')
+        title = match1(content, r'title\s*:\s*"\s*([^"]+)"')
 
     qq_download_by_vid(vid, title, output_dir, merge, info_only)
 

From bd2179ed7689916d6c1ae67d8ea08afc35224d20 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 23 Jan 2016 11:47:08 +0800
Subject: [PATCH 0007/1225] [qq] fix title for some videos

e.g. http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
hope this won't break others, or we should try to recognize the URL
pattern (maybe apply to only URLs with "/cover/")?
---
 src/you_get/extractors/qq.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index b1d8eb0fa3..eb188f9876 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -19,6 +19,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     content = get_html(url)
     vid = match1(content, r'vid\s*:\s*"\s*([^"]+)"')
     title = match1(content, r'title\s*:\s*"\s*([^"]+)"')
+    # try to get the right title for URLs like this:
+    # http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
+    title = matchall(content, [r'title\s*:\s*"\s*([^"]+)"'])[-1]
 
     qq_download_by_vid(vid, title, output_dir, merge, info_only)
 

From 28d04b0ad41a529466b0a77066e52053641f3caf Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 23 Jan 2016 13:06:23 +0800
Subject: [PATCH 0008/1225] [bar] dynamically calculate bar size

or it'll may be too short or too long as 'total_pieces' changes
old one has problems with http://www.letv.com/ptv/vplay/1709438.html
---
 src/you_get/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7e6dced040..ab51bb6c9e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -540,8 +540,8 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     os.rename(temp_filepath, filepath)
 
 class SimpleProgressBar:
-    bar_size = term.get_terminal_size()[1] - 42
-    bar = '{0:>5}% ({1:>5}/{2:<5}MB) ├{3:─<' + str(bar_size) + '}┤[{4}/{5}] {6}'
+    # minus the size of all statically known size in self.bar
+    bar_size = term.get_terminal_size()[1] - 38
 
     def __init__(self, total_size, total_pieces = 1):
         self.displayed = False
@@ -552,6 +552,10 @@ def __init__(self, total_size, total_pieces = 1):
         self.speed = ''
         self.last_updated = time.time()
 
+        total_pieces_len = len(str(total_pieces))
+        self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % (
+            self.bar_size - 2*total_pieces_len, total_pieces_len, total_pieces_len)
+
     def update(self):
         self.displayed = True
         bar_size = self.bar_size

From af588b2f9f63cc6be1bc41c12a677b80dede9e69 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 23 Jan 2016 13:07:18 +0800
Subject: [PATCH 0009/1225] [bar] update bar display at the beginning

or it may not show up itself promptly, leaving the user wondering why it
doesn't start.

I met this problem with http://www.letv.com/ptv/vplay/1709438.html,
which downloads really fast, but I have to wait a few seconds before the
bar shows up.
---
 src/you_get/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index ab51bb6c9e..ebdc6f429d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -691,11 +691,13 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
     if len(urls) == 1:
         url = urls[0]
         print('Downloading %s ...' % tr(output_filename))
+        bar.update()
         url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers)
         bar.done()
     else:
         parts = []
         print('Downloading %s.%s ...' % (tr(title), ext))
+        bar.update()
         for i, url in enumerate(urls):
             filename = '%s[%02d].%s' % (title, i, ext)
             filepath = os.path.join(output_dir, filename)

From d470c8c29aa9910ac733411ab606b4f44f125fef Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 23 Jan 2016 13:10:42 +0800
Subject: [PATCH 0010/1225] [ffmpeg] more check_call replacing wrong 'raise's

---
 src/you_get/processor/ffmpeg.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 6b61ae1837..7262c46218 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -170,11 +170,9 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
-        if subprocess.call(params) == 0:
-            os.remove(output + '.txt')
-            return True
-        else:
-            raise
+        subprocess.check_call(params)
+        os.remove(output + '.txt')
+        return True
 
     for file in files:
         if os.path.isfile(file):
@@ -196,9 +194,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     else:
         params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
 
-    if subprocess.call(params) == 0:
-        for file in files:
-            os.remove(file + '.ts')
-        return True
-    else:
-        raise
+    subprocess.check_call(params)
+    for file in files:
+        os.remove(file + '.ts')
+    return True

From 3941e0d1c67e7597eed930947f05d93d8ce729f0 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sun, 24 Jan 2016 18:46:41 +0800
Subject: [PATCH 0011/1225] [bar] fix bar size

---
 src/you_get/common.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index ebdc6f429d..e147703464 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -540,8 +540,7 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     os.rename(temp_filepath, filepath)
 
 class SimpleProgressBar:
-    # minus the size of all statically known size in self.bar
-    bar_size = term.get_terminal_size()[1] - 38
+    term_size = term.get_terminal_size()[1]
 
     def __init__(self, total_size, total_pieces = 1):
         self.displayed = False
@@ -553,8 +552,10 @@ def __init__(self, total_size, total_pieces = 1):
         self.last_updated = time.time()
 
         total_pieces_len = len(str(total_pieces))
+        # 38 is the size of all statically known size in self.bar
+        self.bar_size = self.term_size - 38 - 2*total_pieces_len
         self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % (
-            self.bar_size - 2*total_pieces_len, total_pieces_len, total_pieces_len)
+            self.bar_size, total_pieces_len, total_pieces_len)
 
     def update(self):
         self.displayed = True

From a9f063f2e802b2ae1ff15abe04969c886f4da5b3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 25 Jan 2016 14:37:29 +0100
Subject: [PATCH 0012/1225] [tumblr] fix #901 (1)

---
 src/you_get/extractors/tumblr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 3969042a02..7f23b15bf3 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -66,7 +66,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if not real_url:
         real_url = r1(r'<source src="([^"]*)"', html)
     if not real_url:
-        iframe_url = r1(r'<iframe [^>]+src=[\'"]([^\'"]*)[\'"]', html)
+        iframe_url = r1(r'<iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
         if iframe_url[:2] == '//': iframe_url = 'http:' + iframe_url
         if re.search(r'player\.vimeo\.com', iframe_url):
             vimeo_download(iframe_url, output_dir, merge=merge, info_only=info_only,

From 13168259f93ed1a63acadace6234e7a25de23b58 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 25 Jan 2016 14:48:55 +0100
Subject: [PATCH 0013/1225] [tumblr] fix #901 (2, embedded vine videos)

---
 src/you_get/extractors/tumblr.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 7f23b15bf3..1fd48940b6 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -6,6 +6,7 @@
 from .universal import *
 from .dailymotion import dailymotion_download
 from .vimeo import vimeo_download
+from .vine import vine_download
 
 def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
@@ -75,6 +76,9 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         elif re.search(r'dailymotion\.com', iframe_url):
             dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
             return
+        elif re.search(r'vine\.co', iframe_url):
+            vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
+            return
         else:
             iframe_html = get_content(iframe_url)
             real_url = r1(r'<source src="([^"]*)"', iframe_html)

From 4ac5978948efbe25088e3721e5348052c56fe144 Mon Sep 17 00:00:00 2001
From: jiemoon <jiemoon2010@gmail.com>
Date: Sun, 31 Jan 2016 15:00:09 +0800
Subject: [PATCH 0014/1225] =?UTF-8?q?Fix=20=E7=A7=92=E6=8B=8D=E9=83=A8?=
 =?UTF-8?q?=E5=88=86=E8=A7=86=E9=A2=91=E6=A0=87=E8=AF=86=E5=8C=85=E5=90=AB?=
 =?UTF-8?q?=E6=8D=A2=E8=A1=8C=E7=AC=A6=E7=AD=89=EF=BC=8C=E5=AF=BC=E8=87=B4?=
 =?UTF-8?q?=E6=AD=A3=E5=88=99=E5=8C=B9=E9=85=8D=E5=BC=82=E5=B8=B8=E7=9A=84?=
 =?UTF-8?q?bug?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/miaopai.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 030e39b45c..235650957e 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -16,14 +16,14 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
             'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
         }
         webpage_url = re.search(r'(http://video.weibo.com/show\?fid=\d{4}:\w{32})\w*', url).group(1) + '&type=mp4'  #mobile
-        
+
         #grab download URL
         a = get_content(webpage_url, headers= fake_headers_mobile , decoded=True)
         url = match1(a, r'<video src="(.*?)\"\W')
-        
+
         #grab title
         b = get_content(webpage_url)  #normal
-        title = match1(b, r'<meta name="description" content="(.*?)\"\W')
+        title = match1(b, r'<meta name="description" content="([\s\S]*?)\"\W')
 
         type_, ext, size = url_info(url)
         print_info(site_info, title, type_, size)

From af8d827d57703b3747ddb0622d283ed820ba75db Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 3 Feb 2016 15:43:36 +0100
Subject: [PATCH 0015/1225] version 0.4.266

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 18dfbd5384..be5056dc23 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.245'
+__version__ = '0.4.266'

From 9c0abc6c64ae7d9c22e68f551b01b874b1ec3b6e Mon Sep 17 00:00:00 2001
From: xyuanmu <xyuanmu@gmail.com>
Date: Thu, 4 Feb 2016 11:48:16 +0800
Subject: [PATCH 0016/1225] fix m5.music.126.net return 404

---
 src/you_get/common.py             | 1 +
 src/you_get/extractors/netease.py | 9 +++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e147703464..6244c57992 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -348,6 +348,7 @@ def url_info(url, faker = False, headers = {}):
     headers = response.headers
 
     type = headers['content-type']
+    if type == 'image/jpg; charset=UTF-8' or type == 'image/jpg' : type = 'audio/mpeg'    #fix for netease
     mapping = {
         'video/3gpp': '3gp',
         'video/f4v': 'flv',
diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 16f671448f..e838878b25 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -94,13 +94,14 @@ def netease_video_download(vinfo, output_dir='.', info_only=False):
 
 def netease_song_download(song, output_dir='.', info_only=False):
     title = "%s. %s" % (song['position'], song['name'])
+    songNet = 'p' + song['mp3Url'].split('/')[2][1:]
 
     if 'hMusic' in song and song['hMusic'] != None:
-        url_best = make_url(song['hMusic']['dfsId'])
+        url_best = make_url(songNet, song['hMusic']['dfsId'])
     elif 'mp3Url' in song:
         url_best = song['mp3Url']
     elif 'bMusic' in song:
-        url_best = make_url(song['bMusic']['dfsId'])
+        url_best = make_url(songNet, song['bMusic']['dfsId'])
 
     netease_download_common(title, url_best,
                             output_dir=output_dir, info_only=info_only)
@@ -165,9 +166,9 @@ def encrypted_id(dfsId):
     return result
 
 
-def make_url(dfsId):
+def make_url(songNet, dfsId):
     encId = encrypted_id(dfsId)
-    mp3_url = "http://m5.music.126.net/%s/%s.mp3" % (encId, dfsId)
+    mp3_url = "http://%s/%s/%s.mp3" % (songNet, encId, dfsId)
     return mp3_url
 
 

From 966840e22a4e48c69a306dcd2d807c2a8a106327 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 4 Feb 2016 20:29:03 +0100
Subject: [PATCH 0017/1225] Revert pull request #898

---
 src/you_get/common.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6244c57992..8bece7c559 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -541,7 +541,8 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     os.rename(temp_filepath, filepath)
 
 class SimpleProgressBar:
-    term_size = term.get_terminal_size()[1]
+    bar_size = term.get_terminal_size()[1] - 42
+    bar = '{0:>5}% ({1:>5}/{2:<5}MB) ├{3:─<' + str(bar_size) + '}┤[{4}/{5}] {6}'
 
     def __init__(self, total_size, total_pieces = 1):
         self.displayed = False
@@ -552,12 +553,6 @@ def __init__(self, total_size, total_pieces = 1):
         self.speed = ''
         self.last_updated = time.time()
 
-        total_pieces_len = len(str(total_pieces))
-        # 38 is the size of all statically known size in self.bar
-        self.bar_size = self.term_size - 38 - 2*total_pieces_len
-        self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % (
-            self.bar_size, total_pieces_len, total_pieces_len)
-
     def update(self):
         self.displayed = True
         bar_size = self.bar_size
@@ -693,13 +688,11 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
     if len(urls) == 1:
         url = urls[0]
         print('Downloading %s ...' % tr(output_filename))
-        bar.update()
         url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers)
         bar.done()
     else:
         parts = []
         print('Downloading %s.%s ...' % (tr(title), ext))
-        bar.update()
         for i, url in enumerate(urls):
             filename = '%s[%02d].%s' % (title, i, ext)
             filepath = os.path.join(output_dir, filename)

From 78801cc496363d99c1d370cc1ad7883d98974c5f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 4 Feb 2016 22:28:25 +0100
Subject: [PATCH 0018/1225] [youtube] fix #911

---
 src/you_get/extractors/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index a1fec359e3..3176a93cc6 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -210,15 +210,18 @@ def prepare(self, **kwargs):
         try:
             caption_tracks = ytplayer_config['args']['caption_tracks'].split(',')
             for ct in caption_tracks:
+                lang = None
                 for i in ct.split('&'):
                     [k, v] = i.split('=')
-                    if k == 'lc': lang = v
+                    if k == 'lc' and lang is None: lang = v
+                    if k == 'v' and v[0] != '.': lang = v # auto-generated
                     if k == 'u': ttsurl = parse.unquote_plus(v)
                 tts_xml = parseString(get_content(ttsurl))
                 transcript = tts_xml.getElementsByTagName('transcript')[0]
                 texts = transcript.getElementsByTagName('text')
                 srt = ""; seq = 0
                 for text in texts:
+                    if text.firstChild is None: continue # empty element
                     seq += 1
                     start = float(text.getAttribute('start'))
                     if text.getAttribute('dur'):

From 07b864dd16130a3ae3ffee2a1315f93ed175c847 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 5 Feb 2016 16:29:00 +0100
Subject: [PATCH 0019/1225] version 0.4.272

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index be5056dc23..a0f7c441b8 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.266'
+__version__ = '0.4.272'

From 9d193d7a36f17a922d698e269695e503c04ff4d7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 5 Feb 2016 16:45:05 +0100
Subject: [PATCH 0020/1225] [README] close #893

---
 README.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index b480f114ad..0380c03760 100644
--- a/README.md
+++ b/README.md
@@ -72,7 +72,13 @@ You may either download the [stable](https://github.com/soimort/you-get/archive/
 Alternatively, run
 
 ```
-$ make install
+$ [sudo] python3 setup.py install
+```
+
+Or
+
+```
+$ python3 setup.py install --user
 ```
 
 to install `you-get` to a permanent path.
@@ -85,7 +91,7 @@ This is the recommended way for all developers, even if you don't often code in
 $ git clone git://github.com/soimort/you-get.git
 ```
 
-Then put the cloned directory into your `PATH`, or run `make install` to install `you-get` to a permanent path.
+Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
 ## Upgrading
 

From 129695b3c6f684b644ff0a1b504bb9a171050c57 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 7 Feb 2016 15:11:25 +0100
Subject: [PATCH 0021/1225] [163] fix #917

---
 src/you_get/extractors/netease.py | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index e838878b25..1fc7c07171 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -129,17 +129,10 @@ def netease_download(url, output_dir = '.', merge = True, info_only = False, **k
         src = r1(r'<source src="([^"]+)"', html) or r1(r'<source type="[^"]+" src="([^"]+)"', html)
 
         if src:
-            sd_url = r1(r'(.+)-mobile.mp4', src) + ".flv"
-            _, _, sd_size = url_info(sd_url)
-
-            hd_url = re.sub('/SD/', '/HD/', sd_url)
-            _, _, hd_size = url_info(hd_url)
-
-            if hd_size > sd_size:
-                url, size = hd_url, hd_size
-            else:
-                url, size = sd_url, sd_size
-            ext = 'flv'
+            url = src
+            _, ext, size = url_info(src)
+            #sd_url = r1(r'(.+)-mobile.mp4', src) + ".flv"
+            #hd_url = re.sub('/SD/', '/HD/', sd_url)
 
         else:
             url = (r1(r'["\'](.+)-list.m3u8["\']', html) or r1(r'["\'](.+).m3u8["\']', html)) + ".mp4"

From 1e2f5de8454ac03d28518d74998a0fe09c9cda11 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 7 Feb 2016 15:13:24 +0100
Subject: [PATCH 0022/1225] [youtube] fix #912

---
 src/you_get/extractors/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 3176a93cc6..70a5f1cb22 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -86,6 +86,7 @@ def get_vid_from_url(url):
         return match1(url, r'youtu\.be/([^/]+)') or \
           match1(url, r'youtube\.com/embed/([^/?]+)') or \
           match1(url, r'youtube\.com/v/([^/?]+)') or \
+          match1(url, r'youtube\.com/watch/([^/?]+)') or \
           parse_query_param(url, 'v') or \
           parse_query_param(parse_query_param(url, 'u'), 'v')
 

From 8c3dc1e11ff740d435426f92a0f4613a6fdf243c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 10 Feb 2016 02:26:43 +0100
Subject: [PATCH 0023/1225] [56] fix #867

---
 src/you_get/extractors/w56.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/w56.py b/src/you_get/extractors/w56.py
index c418067aca..6b9ff0a1be 100644
--- a/src/you_get/extractors/w56.py
+++ b/src/you_get/extractors/w56.py
@@ -4,10 +4,13 @@
 
 from ..common import *
 
+from .sohu import sohu_download
+
 import json
 
 def w56_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
-    info = json.loads(get_html('http://vxml.56.com/json/%s/?src=site' % id))['info']
+    content = json.loads(get_html('http://vxml.56.com/json/%s/?src=site' % id))
+    info = content['info']
     title = title or info['Subject']
     assert title
     hd = info['hd']
@@ -24,6 +27,12 @@ def w56_download_by_id(id, title = None, output_dir = '.', merge = True, info_on
         download_urls([url], title, ext, size, output_dir = output_dir, merge = merge)
 
 def w56_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    content = get_content(url)
+    sohu_url = r1(r"url:\s*'([^']*)'", content)
+    if sohu_url:
+        sohu_download(sohu_url, output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
+
     id = r1(r'http://www.56.com/u\d+/v_(\w+).html', url) or \
          r1(r'http://www.56.com/.*vid-(\w+).html', url)
     w56_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)

From 4e05b55394d659d7f7d8782972aa67634c38857a Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Wed, 10 Feb 2016 16:41:52 +0800
Subject: [PATCH 0024/1225] Revert "Revert pull request #898"

This reverts commit 966840e22a4e48c69a306dcd2d807c2a8a106327.
---
 src/you_get/common.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8bece7c559..6244c57992 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -541,8 +541,7 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     os.rename(temp_filepath, filepath)
 
 class SimpleProgressBar:
-    bar_size = term.get_terminal_size()[1] - 42
-    bar = '{0:>5}% ({1:>5}/{2:<5}MB) ├{3:─<' + str(bar_size) + '}┤[{4}/{5}] {6}'
+    term_size = term.get_terminal_size()[1]
 
     def __init__(self, total_size, total_pieces = 1):
         self.displayed = False
@@ -553,6 +552,12 @@ def __init__(self, total_size, total_pieces = 1):
         self.speed = ''
         self.last_updated = time.time()
 
+        total_pieces_len = len(str(total_pieces))
+        # 38 is the size of all statically known size in self.bar
+        self.bar_size = self.term_size - 38 - 2*total_pieces_len
+        self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % (
+            self.bar_size, total_pieces_len, total_pieces_len)
+
     def update(self):
         self.displayed = True
         bar_size = self.bar_size
@@ -688,11 +693,13 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
     if len(urls) == 1:
         url = urls[0]
         print('Downloading %s ...' % tr(output_filename))
+        bar.update()
         url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers)
         bar.done()
     else:
         parts = []
         print('Downloading %s.%s ...' % (tr(title), ext))
+        bar.update()
         for i, url in enumerate(urls):
             filename = '%s[%02d].%s' % (title, i, ext)
             filepath = os.path.join(output_dir, filename)

From 9fa6ef1f6f248f6aaa24f801bdee14a76e22eb19 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Wed, 10 Feb 2016 16:21:39 +0800
Subject: [PATCH 0025/1225] [bar] don't overflow on large files (>1G)

https://github.com/soimort/you-get/pull/898#issuecomment-180015169
---
 src/you_get/common.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6244c57992..83cd32716d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -554,9 +554,11 @@ def __init__(self, total_size, total_pieces = 1):
 
         total_pieces_len = len(str(total_pieces))
         # 38 is the size of all statically known size in self.bar
-        self.bar_size = self.term_size - 38 - 2*total_pieces_len
-        self.bar = '{0:>5}%% ({1:>5}/{2:<5}MB) ├{3:─<%s}┤[{4:>%s}/{5:>%s}] {6}' % (
-            self.bar_size, total_pieces_len, total_pieces_len)
+        total_str = '%5s' % round(self.total_size / 1048576, 1)
+        total_str_width = max(len(total_str), 5)
+        self.bar_size = self.term_size - 28 - 2*total_pieces_len - 2*total_str_width
+        self.bar = '{:>5}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % (
+            total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len)
 
     def update(self):
         self.displayed = True
@@ -573,7 +575,7 @@ def update(self):
         else:
             plus = ''
         bar = '█' * dots + plus
-        bar = self.bar.format(percent, round(self.received / 1048576, 1), round(self.total_size / 1048576, 1), bar, self.current_piece, self.total_pieces, self.speed)
+        bar = self.bar.format(percent, round(self.received / 1048576, 1), bar, self.current_piece, self.total_pieces, self.speed)
         sys.stdout.write('\r' + bar)
         sys.stdout.flush()
 

From af42e7cedcbe67dcb14e41ff11480c1572575d20 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Wed, 10 Feb 2016 16:23:13 +0800
Subject: [PATCH 0026/1225] [bar] show "100%" instead of "100.0%"

---
 src/you_get/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 83cd32716d..aaa9457ba7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -556,15 +556,15 @@ def __init__(self, total_size, total_pieces = 1):
         # 38 is the size of all statically known size in self.bar
         total_str = '%5s' % round(self.total_size / 1048576, 1)
         total_str_width = max(len(total_str), 5)
-        self.bar_size = self.term_size - 28 - 2*total_pieces_len - 2*total_str_width
-        self.bar = '{:>5}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % (
+        self.bar_size = self.term_size - 27 - 2*total_pieces_len - 2*total_str_width
+        self.bar = '{:>4}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % (
             total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len)
 
     def update(self):
         self.displayed = True
         bar_size = self.bar_size
         percent = round(self.received * 100 / self.total_size, 1)
-        if percent > 100:
+        if percent >= 100:
             percent = 100
         dots = bar_size * int(percent) // 100
         plus = int(percent) - dots // bar_size * 100

From bf59adcd8b6991ae873005eb15b16fec91d60792 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Wed, 10 Feb 2016 16:26:10 +0800
Subject: [PATCH 0027/1225] [bar] handle speed >= 1GiB/s nicely

I suppose there won't be >= 1TiB/s speed in the near future.
---
 src/you_get/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index aaa9457ba7..08c9b45a76 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -583,8 +583,10 @@ def update_received(self, n):
         self.received += n
         time_diff = time.time() - self.last_updated
         bytes_ps = n / time_diff if time_diff else 0
-        if bytes_ps >= 1048576:
-            self.speed = '{:4.0f} MB/s'.format(bytes_ps / 1048576)
+        if bytes_ps >= 1024 ** 3:
+            self.speed = '{:4.0f} GB/s'.format(bytes_ps / 1024 ** 3)
+        elif bytes_ps >= 1024 ** 2:
+            self.speed = '{:4.0f} MB/s'.format(bytes_ps / 1024 ** 2)
         elif bytes_ps >= 1024:
             self.speed = '{:4.0f} kB/s'.format(bytes_ps / 1024)
         else:

From 321572404b2c22dc85d3ca7b982f5b7cdfca19c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 11 Feb 2016 01:07:34 +0100
Subject: [PATCH 0028/1225] [755] fix #919

---
 src/you_get/extractors/nanagogo.py | 84 +++++++++++-------------------
 1 file changed, 31 insertions(+), 53 deletions(-)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index 731ea0a36c..df050390e3 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -5,59 +5,37 @@
 from ..common import *
 
 def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html(url)
-    title = r1(r'<meta property="og:title" content="([^"]*)"', html)
-    postId = r1(r'postId\s*:\s*"([^"]*)"', html)
-    title += ' - ' + postId
-    try: # extract direct video
-        source = r1(r'<meta property="og:video" content="([^"]*)"', html)
-        mime, ext, size = url_info(source)
-
-        print_info(site_info, title, mime, size)
-        if not info_only:
-            download_urls([source], title, ext, size, output_dir, merge=merge)
-
-    except: # official API
-        talkId = r1(r'talkId\s*:\s*"([^"]*)"', html)
-        apiUrl = 'http://7gogo.jp/api/talk/post/detail/%s/%s' % (talkId, postId)
-        info = json.loads(get_content(apiUrl))
-        images = []
-        for post in info['posts']:
-            for item in post['body']:
-                if 'movieUrlHq' in item:
-                    url = item['movieUrlHq']
-                    name = title
-                    _, ext, size = url_info(url)
-                    images.append({'title': name,
-                                   'url': url,
-                                   'ext': ext,
-                                   'size': size})
-
-                elif 'image' in item:
-                    url = item['image']
-                    name = title
-                    #filename = parse.unquote(url.split('/')[-1])
-                    #name = '.'.join(filename.split('.')[:-1])
-                    #ext = filename.split('.')[-1]
-                    #size = int(get_head(url)['Content-Length'])
-                    _, ext, size = url_info(url)
-                    images.append({'title': name,
-                                   'url': url,
-                                   'ext': ext,
-                                   'size': size})
-
-        size = sum([i['size'] for i in images])
-        print_info(site_info, title, ext, size)
-
-        if not info_only:
-            for i in images:
-                title = i['title']
-                ext = i['ext']
-                size = i['size']
-                url = i['url']
-                print_info(site_info, title, ext, size)
-                download_urls([url], title, ext, size,
-                              output_dir=output_dir)
+    talk_id = r1(r'7gogo.jp/([^/]+)/', url)
+    post_id = r1(r'7gogo.jp/[^/]+/(\d+)', url)
+    title = '%s_%s' % (talk_id, post_id)
+    api_url = 'https://api.7gogo.jp/web/v2/talks/%s/posts/%s' % (talk_id, post_id)
+    info = json.loads(get_content(api_url))
+
+    items = []
+    for i in info['data']['posts']['post']['body']:
+        if 'image' in i:
+            image_url = i['image']
+            _, ext, size = url_info(image_url)
+            items.append({'title': title,
+                          'url':   image_url,
+                          'ext':   ext,
+                          'size':  size})
+        elif 'movieUrlHq' in i:
+            movie_url = i['movieUrlHq']
+            _, ext, size = url_info(movie_url)
+            items.append({'title': title,
+                          'url':   movie_url,
+                          'ext':   ext,
+                          'size':  size})
+
+    size = sum([i['size'] for i in items])
+    print_info(site_info, title, ext, size)
+    if not info_only:
+        for i in items:
+            print_info(site_info, i['title'], i['ext'], i['size'])
+            download_urls([i['url']], i['title'], i['ext'], i['size'],
+                          output_dir=output_dir,
+                          merge=merge)
 
 site_info = "7gogo.jp"
 download = nanagogo_download

From 051895c901d9cd4f7325674cb746604ffd62eeea Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Tue, 16 Feb 2016 00:29:40 -0500
Subject: [PATCH 0029/1225] [THVideo] Add Support

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/thvideo.py  | 83 ++++++++++++++++++++++++++++++
 3 files changed, 85 insertions(+)
 create mode 100644 src/you_get/extractors/thvideo.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8bece7c559..ef0de059eb 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -61,6 +61,7 @@
     'soundcloud'       : 'soundcloud',
     'ted'              : 'ted',
     'theplatform'      : 'theplatform',
+    'thvideo'          : 'thvideo',
     'tucao'            : 'tucao',
     'tudou'            : 'tudou',
     'tumblr'           : 'tumblr',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 7b3c6a7162..dfd45596bb 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -53,6 +53,7 @@
 from .soundcloud import *
 from .suntv import *
 from .theplatform import *
+from .thvideo import *
 from .tucao import *
 from .tudou import *
 from .tumblr import *
diff --git a/src/you_get/extractors/thvideo.py b/src/you_get/extractors/thvideo.py
new file mode 100644
index 0000000000..2317b6f5b7
--- /dev/null
+++ b/src/you_get/extractors/thvideo.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+
+__all__ = ['thvideo_download']
+
+from ..common import *
+from xml.dom.minidom import parseString
+
+#----------------------------------------------------------------------
+def thvideo_cid_to_url(cid, p):
+    """int,int->list
+    From Biligrab."""
+    interface_url = 'http://thvideo.tv/api/playurl.php?cid={cid}-{p}'.format(cid = cid, p = p)
+    data = get_content(interface_url)
+    rawurl = []
+    dom = parseString(data)
+    
+    for node in dom.getElementsByTagName('durl'):
+        url = node.getElementsByTagName('url')[0]
+        rawurl.append(url.childNodes[0].data)
+    return rawurl
+
+#----------------------------------------------------------------------
+def th_video_get_title(url, p):
+    """"""
+    if re.match(r'http://thvideo.tv/v/\w+', url):
+        html = get_content(url)
+        title = match1(html, r'<meta property="og:title" content="([^"]*)"').strip()
+        
+        video_list = match1(html, r'<li>cid=(.+)</li>').split('**')
+        
+        if int(p) > 0:  #not the 1st P or multi part
+            title = title + ' - ' + [i.split('=')[-1:][0].split('|')[1] for i in video_list][p]
+            
+    return title
+
+#----------------------------------------------------------------------
+def thvideo_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    if re.match(r'http://thvideo.tv/v/\w+', url):
+        if 'p' in kwargs and kwargs['p']:
+            p = kwargs['p']
+        else:
+            p = int(match1(url, r'http://thvideo.tv/v/th\d+#(\d+)'))
+            p -= 1
+            
+            if not p or p < 0:
+                p = 0
+        
+        if 'title' in kwargs and kwargs['title']:
+            title = kwargs['title']
+        else:
+            title = th_video_get_title(url, p)
+        
+        cid = match1(url, r'http://thvideo.tv/v/th(\d+)')
+        
+        type_ = ''
+        size = 0
+        urls = thvideo_cid_to_url(cid, p)
+        
+        for url in urls:
+            _, type_, temp = url_info(url)
+            size += temp
+        
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
+#----------------------------------------------------------------------
+def thvideo_download_playlist(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    """"""
+    if re.match(r'http://thvideo.tv/v/\w+', url):
+        html = get_content(url)
+        video_list = match1(html, r'<li>cid=(.+)</li>').split('**')
+        
+        title_base = th_video_get_title(url, 0)
+        for p, v in video_list:
+            part_title = [i.split('=')[-1:][0].split('|')[1] for i in video_list][p]
+            title = title_base + part_title
+            thvideo_download(url, output_dir, merge, 
+                            info_only, p = p, title = title)
+
+site_info = "THVideo"
+download = thvideo_download
+download_playlist = thvideo_download_playlist

From 94fbbf4d0e6d3d4d8abbb1f08b84940793a1e3c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Feb 2016 04:23:43 +0100
Subject: [PATCH 0030/1225] add .github/ISSUE_TEMPLATE.md

- See <https://github.com/blog/2111-issue-and-pull-request-templates>
---
 .github/ISSUE_TEMPLATE.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
new file mode 100644
index 0000000000..d6a52e5946
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE.md
@@ -0,0 +1,17 @@
+In case of any encountered problem, always check your network status first. That is, please ensure the video you want to download can be streamed properly in your web browser.
+
+* Keep in mind that some videos on some hosting sites may have a region restriction, e.g., Youku is blocking access to some videos from IP addresses outside mainland China, and YouTube is also blocking some videos in Germany.
+
+Please include:
+
+* Your exact command line, like `you-get -i "www.youtube.com/watch?v=sGwy8DsUJ4M"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+
+* Your full console output.
+
+* If you executed the command and got no response, please re-run the command with `--debug`, kill the process with keyboard shortcut `Ctrl-C` and include the full console output.
+
+* The output of `you-get --version`, or `git rev-parse HEAD` -- if you are using a Git version (but always remember to keep up-to-date!)
+
+* The output of `python --version`.
+
+* If possible, you may include your IP address and proxy setting information as well.

From 26ed950f16b6754b3816140421037d51502dec76 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Feb 2016 05:25:38 +0100
Subject: [PATCH 0031/1225] update .github/ISSUE_TEMPLATE.md

---
 .github/ISSUE_TEMPLATE.md | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index d6a52e5946..5ec3a5e1f8 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,17 +1,17 @@
-In case of any encountered problem, always check your network status first. That is, please ensure the video you want to download can be streamed properly in your web browser.
+Please make sure these boxes are checked before submitting your issue – thank you!
 
-* Keep in mind that some videos on some hosting sites may have a region restriction, e.g., Youku is blocking access to some videos from IP addresses outside mainland China, and YouTube is also blocking some videos in Germany.
+- [ ] You can actually watch the video in your browser, but not download them with `you-get`.
+- [ ] Your `you-get` is up-to-date.
+- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or <https://github.com/soimort/you-get/wiki/Known-Bugs>.
+- [ ] The issue (or question) is really about `you-get`, not about some other code or project.
 
-Please include:
+Run the command with the `--debug` option, and paste the full output inside the fences:
 
-* Your exact command line, like `you-get -i "www.youtube.com/watch?v=sGwy8DsUJ4M"`. A common mistake is not to escape the `&`. Putting URLs in quotes should solve this problem.
+```
+[PASTE IN ME]
+```
 
-* Your full console output.
+If there's anything else you would like to say (e.g. in case your issue is not about downloading a specific video; it might as well be a general discussion or proposal for a new feature), fill in the box below; otherwise, you may want to post an emoji or meme instead:
 
-* If you executed the command and got no response, please re-run the command with `--debug`, kill the process with keyboard shortcut `Ctrl-C` and include the full console output.
-
-* The output of `you-get --version`, or `git rev-parse HEAD` -- if you are using a Git version (but always remember to keep up-to-date!)
-
-* The output of `python --version`.
-
-* If possible, you may include your IP address and proxy setting information as well.
+> [WRITE SOMETHING]
+> [OR HAVE SOME :icecream:!]

From c192708fbbef237c4012a9776946cfdbd35e6150 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Feb 2016 06:37:21 +0100
Subject: [PATCH 0032/1225] .github/PULL_REQUEST_TEMPLATE.md

---
 .github/PULL_REQUEST_TEMPLATE.md | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 .github/PULL_REQUEST_TEMPLATE.md

diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
new file mode 100644
index 0000000000..c57b2816c3
--- /dev/null
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -0,0 +1,22 @@
+**(PLEASE DELETE ALL THESE AFTER READING)**
+
+Thank you for the pull request! `you-get` is a growing open source project, which would not have been possible without contributors like you.
+
+Here are some simple rules to follow, please recheck them before sending the pull request:
+
+- [ ] If you want to propose two or more unrelated patches, please open separate pull requests for them, instead of one;
+- [ ] All pull requests should be based upon the latest `develop` branch;
+- [ ] Name your branch (from which you will send the pull request) properly; use a meaningful name like `add-this-shining-feature` rather than just `develop`;
+- [ ] All commit messages, as well as comments in code, should be written in understandable English.
+
+As a contributor, you must be aware that
+
+- [ ] You agree to contribute your code to this project, under the terms of the MIT license, so that any person may freely use or redistribute them; of course, you will still reserve the copyright for your own authorship.
+- [ ] You may not contribute any code not authored by yourself, unless they are licensed under either public domain or the MIT license, literally.
+
+Not all pull requests can eventually be merged. I consider merged / unmerged patches as equally important for the community: as long as you think a patch would be helpful, someone else might find it helpful, too, therefore they could take your fork and benefit in some way. In any case, I would like to thank you in advance for taking your time to contribute to this project.
+
+Cheers,
+Mort
+
+**(PLEASE REPLACE ALL ABOVE WITH A DETAILED DESCRIPTION OF YOUR PULL REQUEST)**

From aabf30641bbb5bba535a98a7e5cfe4d0aea29976 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Feb 2016 15:50:14 +0100
Subject: [PATCH 0033/1225] [tudou] fix #925

---
 src/you_get/extractors/tudou.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index 9f4f124545..d6a7f815c8 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -9,12 +9,11 @@ def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only
     data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
     temp = max([data[i] for i in data if 'size' in data[i][0]], key=lambda x:sum([part['size'] for part in x]))
     vids, size = [t["k"] for t in temp], sum([t["size"] for t in temp])
-    urls = [[n.firstChild.nodeValue.strip()
-             for n in
-                parseString(
-                    get_html('http://ct.v2.tudou.com/f?id=%s' % vid))
-                .getElementsByTagName('f')][0]
-            for vid in vids]
+
+    urls = []
+    for vid in vids:
+        for i in parseString(get_html('http://ct.v2.tudou.com/f?id=%s' % vid)).getElementsByTagName('f'):
+            urls.append(i.firstChild.nodeValue.strip())
 
     ext = r1(r'http://[\w.]*/(\w+)/[\w.]*', urls[0])
 

From ab3fbe0502fbafc0abf995cf7071449dabc627c6 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Fri, 19 Feb 2016 02:12:39 -0500
Subject: [PATCH 0034/1225] [YIxia-Miaopai] Fix #926

---
 src/you_get/extractors/yixia.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index 06433bdedb..fa92dfe674 100755
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -54,9 +54,9 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         if re.match(r'http://www.miaopai.com/show/channel/\w+', url):  #PC
             scid = match1(url, r'http://www.miaopai.com/show/channel/(\w+)')
         elif re.match(r'http://www.miaopai.com/show/\w+', url):  #PC
-            scid = match1(url, r'http://www.miaopai.com/show/(\w+)')
+            scid = match1(url, r'http://www.miaopai.com/show/(.+)\.htm')
         elif re.match(r'http://m.miaopai.com/show/channel/\w+', url):  #Mobile
-            scid = match1(url, r'http://m.miaopai.com/show/channel/(\w+)')
+            scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)\.htm')
     
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu
         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid

From 2d71b96847ac4cfc958f6fc2a0ae3687c98596ab Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Fri, 19 Feb 2016 02:20:20 -0500
Subject: [PATCH 0035/1225] [Yixia-Miaopai]Updating more pattern

---
 src/you_get/extractors/yixia.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index fa92dfe674..ca5c4bd6ab 100755
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -52,7 +52,7 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         site_info = "Yixia Miaopai"
         
         if re.match(r'http://www.miaopai.com/show/channel/\w+', url):  #PC
-            scid = match1(url, r'http://www.miaopai.com/show/channel/(\w+)')
+            scid = match1(url, r'http://www.miaopai.com/show/channel/(.+)\.htm')
         elif re.match(r'http://www.miaopai.com/show/\w+', url):  #PC
             scid = match1(url, r'http://www.miaopai.com/show/(.+)\.htm')
         elif re.match(r'http://m.miaopai.com/show/channel/\w+', url):  #Mobile

From b3623c3404ac4ff6e05e6b413f5a7d489e4e68ec Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 19 Feb 2016 14:32:51 +0100
Subject: [PATCH 0036/1225] version 0.4.293

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index a0f7c441b8..018a1970f7 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.272'
+__version__ = '0.4.293'

From 96e2c3a93d70657dae76ae7b4719a04857856b50 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 20 Feb 2016 09:57:39 +0100
Subject: [PATCH 0037/1225] update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 0380c03760..7c1b211b5c 100644
--- a/README.md
+++ b/README.md
@@ -359,6 +359,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
 | QQ<br/>腾讯视频 | <http://v.qq.com/>      |✓| | |
 | 阡陌视频 | <http://qianmo.com/>           |✓| | |
+| THVideo  | <http://thvideo.tv/>           |✓| | |
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
 | Sohu<br/>搜狐视频 | <http://tv.sohu.com/> |✓| | |
 | 天天动听 | <http://www.dongting.com/>     | | |✓|

From 2948847caf4c822ff7e22a71f68b8da78b0123f9 Mon Sep 17 00:00:00 2001
From: jiemoon <jiemoon2010@gmail.com>
Date: Wed, 24 Feb 2016 15:41:47 +0800
Subject: [PATCH 0038/1225] [QQ] update the url of get video info

---
 src/you_get/extractors/qq.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 0a007c0be1..a0d80664c5 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,10 +5,12 @@
 from ..common import *
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    api = "http://vv.video.qq.com/geturl?otype=json&vid=%s" % vid
+    api = "http://h5vv.video.qq.com/getinfo?otype=json&vid=%s" % vid
     content = get_html(api)
     output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
-    url = output_json['vd']['vi'][0]['url']
+    url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']
+    fvkey = output_json['vl']['vi'][0]['fvkey']
+    url = '%s/%s.mp4?vkey=%s' % ( url, vid, fvkey )
     _, ext, size = url_info(url, faker=True)
 
     print_info(site_info, title, ext, size)

From 92a6efcbfe26386f4be31aa5ac4ec78a19b5d40f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Feb 2016 20:14:59 +0100
Subject: [PATCH 0039/1225] [acfun] fix exception when caption not present in
 youku source

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 50eb9dfc89..b120605687 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -24,7 +24,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     if sourceType == 'sina':
         sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'youku':
-        youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
     elif sourceType == 'tudou':
         tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'qq':

From 3931c4be0341428320020094fc78a5a24c050b82 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Feb 2016 01:31:26 +0100
Subject: [PATCH 0040/1225] [755] skip stamp images

---
 src/you_get/extractors/nanagogo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index df050390e3..6f94043927 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -15,6 +15,7 @@ def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     for i in info['data']['posts']['post']['body']:
         if 'image' in i:
             image_url = i['image']
+            if image_url[:2] == '//': continue # skip stamp images
             _, ext, size = url_info(image_url)
             items.append({'title': title,
                           'url':   image_url,

From abee36003bdefa60f16dec3817d45c76581a6ceb Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Fri, 26 Feb 2016 10:51:13 -0500
Subject: [PATCH 0041/1225] [.github]Chinese translation

---
 .github/ISSUE_TEMPLATE.md        | 24 ++++++++++++++++++++++--
 .github/PULL_REQUEST_TEMPLATE.md | 26 ++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5ec3a5e1f8..5cf8212b3a 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -1,8 +1,8 @@
 Please make sure these boxes are checked before submitting your issue – thank you!
 
-- [ ] You can actually watch the video in your browser, but not download them with `you-get`.
+- [ ] You can actually watch the video in your browser or mobile application, but not download them with `you-get`.
 - [ ] Your `you-get` is up-to-date.
-- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or <https://github.com/soimort/you-get/wiki/Known-Bugs>.
+- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or <https://github.com/soimort/you-get/wiki/Known-Bugs>. If so, please add your comments under the existing issue.
 - [ ] The issue (or question) is really about `you-get`, not about some other code or project.
 
 Run the command with the `--debug` option, and paste the full output inside the fences:
@@ -15,3 +15,23 @@ If there's anything else you would like to say (e.g. in case your issue is not a
 
 > [WRITE SOMETHING]
 > [OR HAVE SOME :icecream:!]
+
+汉语翻译最终日期：2016年02月26日
+
+在提交前，请确保您已经检查了以下内容!
+
+- [ ] 你可以在浏览器或移动端中观看视频，但不能使用`you-get`下载.
+- [ ] 您的`you-get`为最新版.
+- [ ] 您的问题没有在<https://github.com/soimort/you-get/issues> 或 <https://github.com/soimort/you-get/wiki/Known-Bugs> 报告，否则请在原有issue下报告.
+- [ ] 本问题确实关于`you-get`, 而不是其他项目.
+
+请使用`--debug`运行，并将输出粘贴在下面:
+
+```
+[在这里粘贴完整日志]
+```
+
+如果您有其他附言，例如问题只在某个视频发生，或者是一般性讨论或者提出新功能，请在下面添加;或者您可以卖个萌:
+
+> [您的内容]
+> [舔 :icecream:!]
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index c57b2816c3..79a43f6bc4 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -20,3 +20,29 @@ Cheers,
 Mort
 
 **(PLEASE REPLACE ALL ABOVE WITH A DETAILED DESCRIPTION OF YOUR PULL REQUEST)**
+
+
+汉语翻译最后日期：2016年02月26日
+
+**(阅读后请删除所有内容)**
+
+感谢您的pull request! `you-get`是稳健成长的开源项目，感谢您的贡献.
+
+以下简单检查项目望您复查:
+
+- [ ] 如果您预计提出两个或更多不相关补丁，请为每个使用不同的pull requests，而不是单一;
+- [ ] 所有的pull requests应基于最新的`develop`分支;
+- [ ] 您预计提出pull requests的分支应有有意义名称，例如`add-this-shining-feature`而不是`develop`;
+- [ ] 所有的提交信息与代码中注释应使用可理解的英语.
+
+作为贡献者，您需要知悉
+
+- [ ] 您同意在MIT协议下贡献代码，以便任何人自由使用或分发;当然，你仍旧保留代码的著作权
+- [ ] 你不得贡献非自己编写的代码，除非其属于公有领域或使用MIT协议.
+
+不是所有的pull requests都会被合并,然而我认为合并/不合并的补丁一样重要：如果您认为补丁重要，其他人也有可能这么认为，那么他们可以从你的fork中提取工作并获益。无论如何，感谢您费心对本项目贡献.
+
+祝好,
+Mort
+
+**(请将本内容完整替换为PULL REQUEST的详细内容)**

From 9bb06c45b444dac2e65fab65c630fbdd9928eb37 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 27 Feb 2016 22:45:08 +0100
Subject: [PATCH 0042/1225] [google+] extract multiple photos in a post

---
 src/you_get/extractors/google.py | 71 +++++++++++++++++---------------
 1 file changed, 38 insertions(+), 33 deletions(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 12bc42de6c..a2cc025d31 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -48,45 +48,50 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
     if service == 'plus': # Google Plus
 
-        if not re.search(r'plus.google.com/photos/[^/]*/albums/\d+/\d+', url):
-            html = get_html(parse.unquote(url))
-            url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
-            title = r1(r'<title>([^<\n]+)', html)
-        else:
-            title = None
-
-        html = get_html(url)
-        temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
-        temp = sorted(temp, key = lambda x : fmt_level[x[0]])
-        real_urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
-
-        if title is None:
-            post_url = r1(r'"(https://plus.google.com/[^/]+/posts/[^"]*)"', html)
-            post_author = r1(r'/\+([^/]+)/posts', post_url)
-            if post_author:
-                post_url = "https://plus.google.com/+%s/posts/%s" % (parse.quote(post_author), r1(r'posts/(.+)', post_url))
-            post_html = get_html(post_url)
-            title = r1(r'<title[^>]*>([^<\n]+)', post_html)
-
-        if title is None:
-            response = request.urlopen(request.Request(real_url))
-            if response.headers['content-disposition']:
-                filename = parse.unquote(r1(r'filename="?(.+)"?', response.headers['content-disposition'])).split('.')
-                title = ''.join(filename[:-1])
+        # attempt to extract images first
+        html = get_html(parse.unquote(url))
+        real_urls = []
+        for src in re.findall(r'src="([^"]+)"[^>]*itemprop="image"', html):
+            t = src.split('/')
+            t[0], t[-2] = t[0] or 'https:', 's0-d'
+            u = '/'.join(t)
+            real_urls.append(u)
+        post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
+        post_id = r1(r'/posts/([^"]+)', html)
+        title = post_date + "_" + post_id
 
         if not real_urls:
-            # extract the image
-            # FIXME: download multple images / albums
-            real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
-            post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
-            post_id = r1(r'/posts/([^"]+)', html)
-            title = post_date + "_" + post_id
+
+            if not re.search(r'plus.google.com/photos/[^/]*/albums/\d+/\d+', url):
+                html = get_html(parse.unquote(url))
+                url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
+                title = r1(r'<title>([^<\n]+)', html)
+            else:
+                title = None
+
+            html = get_html(url)
+            temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
+            temp = sorted(temp, key = lambda x : fmt_level[x[0]])
+            real_urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
+
+            if title is None:
+                post_url = r1(r'"(https://plus.google.com/[^/]+/posts/[^"]*)"', html)
+                post_author = r1(r'/\+([^/]+)/posts', post_url)
+                if post_author:
+                    post_url = "https://plus.google.com/+%s/posts/%s" % (parse.quote(post_author), r1(r'posts/(.+)', post_url))
+                post_html = get_html(post_url)
+                title = r1(r'<title[^>]*>([^<\n]+)', post_html)
+
+            if title is None:
+                response = request.urlopen(request.Request(real_url))
+                if response.headers['content-disposition']:
+                    filename = parse.unquote(r1(r'filename="?(.+)"?', response.headers['content-disposition'])).split('.')
+                    title = ''.join(filename[:-1])
 
         for (i, real_url) in enumerate(real_urls):
             title_i = "%s[%s]" % (title, i) if len(real_urls) > 1 else title
             type, ext, size = url_info(real_url)
-            if ext is None:
-                ext = 'mp4'
+            if ext is None: ext = 'mp4'
 
             print_info(site_info, title_i, ext, size)
             if not info_only:

From 145941fcc24009621d234e804f60e285efdee5e6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 27 Feb 2016 22:45:37 +0100
Subject: [PATCH 0043/1225] [google+] support /photos/.../albums/... urls

---
 src/you_get/extractors/google.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index a2cc025d31..0198a4b859 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -56,6 +56,8 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
             t[0], t[-2] = t[0] or 'https:', 's0-d'
             u = '/'.join(t)
             real_urls.append(u)
+        if real_urls is None:
+            real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
         post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id

From ab4ba75d2f109311c8fa36570617394e5fe15904 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 27 Feb 2016 22:53:48 +0100
Subject: [PATCH 0044/1225] [google+] add TBD comments

---
 src/you_get/extractors/google.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 0198a4b859..fdaa9d463e 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -49,6 +49,8 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
     if service == 'plus': # Google Plus
 
         # attempt to extract images first
+        # TBD: posts with > 4 images
+        # TBD: album links
         html = get_html(parse.unquote(url))
         real_urls = []
         for src in re.findall(r'src="([^"]+)"[^>]*itemprop="image"', html):
@@ -56,7 +58,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
             t[0], t[-2] = t[0] or 'https:', 's0-d'
             u = '/'.join(t)
             real_urls.append(u)
-        if real_urls is None:
+        if not real_urls:
             real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
         post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
         post_id = r1(r'/posts/([^"]+)', html)

From 3493437721882e68eee877373a5ce09065d5c159 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 27 Feb 2016 23:25:51 +0100
Subject: [PATCH 0045/1225] [google+] fix support for videos (image/gif is not
 what we want)

---
 src/you_get/extractors/google.py | 29 ++++++++++++-----------------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index fdaa9d463e..1cd63aabc9 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -64,33 +64,28 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id
 
-        if not real_urls:
-
-            if not re.search(r'plus.google.com/photos/[^/]*/albums/\d+/\d+', url):
-                html = get_html(parse.unquote(url))
-                url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
-                title = r1(r'<title>([^<\n]+)', html)
-            else:
-                title = None
-
+        try:
+            url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
             html = get_html(url)
             temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
             temp = sorted(temp, key = lambda x : fmt_level[x[0]])
-            real_urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
+            urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
+            assert urls
+            real_urls = urls # Look ma, there's really a video!
 
-            if title is None:
-                post_url = r1(r'"(https://plus.google.com/[^/]+/posts/[^"]*)"', html)
-                post_author = r1(r'/\+([^/]+)/posts', post_url)
-                if post_author:
-                    post_url = "https://plus.google.com/+%s/posts/%s" % (parse.quote(post_author), r1(r'posts/(.+)', post_url))
-                post_html = get_html(post_url)
-                title = r1(r'<title[^>]*>([^<\n]+)', post_html)
+            post_url = r1(r'"(https://plus.google.com/[^/]+/posts/[^"]*)"', html)
+            post_author = r1(r'/\+([^/]+)/posts', post_url)
+            if post_author:
+                post_url = "https://plus.google.com/+%s/posts/%s" % (parse.quote(post_author), r1(r'posts/(.+)', post_url))
+            post_html = get_html(post_url)
+            title = r1(r'<title[^>]*>([^<\n]+)', post_html)
 
             if title is None:
                 response = request.urlopen(request.Request(real_url))
                 if response.headers['content-disposition']:
                     filename = parse.unquote(r1(r'filename="?(.+)"?', response.headers['content-disposition'])).split('.')
                     title = ''.join(filename[:-1])
+        except: pass
 
         for (i, real_url) in enumerate(real_urls):
             title_i = "%s[%s]" % (title, i) if len(real_urls) > 1 else title

From a732c15e0e0dfb9c27c9e3b86e5f0c25956019d3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 27 Feb 2016 23:31:20 +0100
Subject: [PATCH 0046/1225] [google+] complete itag: 102

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 1cd63aabc9..18483920ca 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -12,7 +12,7 @@
     {'itag': 38, 'container': 'MP4', 'video_resolution': '3072p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3.5-5', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
     {'itag': 46, 'container': 'WebM', 'video_resolution': '1080p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
     {'itag': 37, 'container': 'MP4', 'video_resolution': '1080p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3-4.3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
-    {'itag': 102, 'container': '', 'video_resolution': '', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '2', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+    {'itag': 102, 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '2', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
     {'itag': 45, 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': '', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': '', 'audio_bitrate': ''},
     {'itag': 22, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
     {'itag': 84, 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-2.9', 'audio_encoding': 'AAC', 'audio_bitrate': '152'},

From 73f5bbc998842e3053b2ec18f6359b2b47ad862b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Feb 2016 15:12:53 +0100
Subject: [PATCH 0047/1225] version 0.4.306

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 018a1970f7..68443fba45 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.293'
+__version__ = '0.4.306'

From efcec113768c3273b1d10d26b076f20a303a079c Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Mon, 29 Feb 2016 17:22:47 -0500
Subject: [PATCH 0048/1225] [Miaopai]Add Weibo URL pattern without login

---
 src/you_get/extractors/miaopai.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 235650957e..9f195baa08 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -5,7 +5,7 @@
 from ..common import *
 import urllib.error
 
-def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+def miaopai_download_by_url(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
     if re.match(r'http://video.weibo.com/show\?fid=(\d{4}:\w{32})\w*', url):
         fake_headers_mobile = {
@@ -30,6 +30,14 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
         if not info_only:
             download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
+#----------------------------------------------------------------------
+def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    """"""
+    if re.match(r'http://video.weibo.com/show\?fid=(\d{4}:\w{32})\w*', url):
+        miaopai_download_by_url(url, output_dir, merge, info_only)
+    elif re.match(r'http://weibo.com/p/230444\w+', url):
+        _fid = match1(url, r'http://weibo.com/p/230444(\w+)')
+        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid))
 
 site_info = "miaopai"
 download = miaopai_download

From 5aa7b87dcecb98be161c6592c7e0291cb7e789f2 Mon Sep 17 00:00:00 2001
From: Sleaze <root@dev.null>
Date: Mon, 29 Feb 2016 15:59:04 -0800
Subject: [PATCH 0049/1225] Support for tumblr-hosted videos.

---
 src/you_get/extractors/tumblr.py | 33 ++++++++++++++++++--------------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 1fd48940b6..fea061cecd 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -67,21 +67,26 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if not real_url:
         real_url = r1(r'<source src="([^"]*)"', html)
     if not real_url:
-        iframe_url = r1(r'<iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
-        if iframe_url[:2] == '//': iframe_url = 'http:' + iframe_url
-        if re.search(r'player\.vimeo\.com', iframe_url):
-            vimeo_download(iframe_url, output_dir, merge=merge, info_only=info_only,
-                           referer='http://tumblr.com/', **kwargs)
-            return
-        elif re.search(r'dailymotion\.com', iframe_url):
-            dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
-            return
-        elif re.search(r'vine\.co', iframe_url):
-            vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
-            return
+        iframe_url = r1(r'<[^>]+tumblr_video_container[^>]+><iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
+        if len(iframe_url) > 0:
+            iframe_html = get_content(iframe_url, headers=fake_headers)
+            real_url = r1(r'<video[^>]*>[\n ]*<source[^>]+src=[\'"]([^\'"]*)[\'"]', iframe_html)
         else:
-            iframe_html = get_content(iframe_url)
-            real_url = r1(r'<source src="([^"]*)"', iframe_html)
+            iframe_url = r1(r'<iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
+            if iframe_url[:2] == '//': iframe_url = 'http:' + iframe_url
+            if re.search(r'player\.vimeo\.com', iframe_url):
+                vimeo_download(iframe_url, output_dir, merge=merge, info_only=info_only,
+                               referer='http://tumblr.com/', **kwargs)
+                return
+            elif re.search(r'dailymotion\.com', iframe_url):
+                dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
+                return
+            elif re.search(r'vine\.co', iframe_url):
+                vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
+                return
+            else:
+                iframe_html = get_content(iframe_url)
+                real_url = r1(r'<source src="([^"]*)"', iframe_html)
 
     title = unescape_html(r1(r'<meta property="og:title" content="([^"]*)" />', html) or
         r1(r'<meta property="og:description" content="([^"]*)" />', html) or

From 81a81b0abeb1b3d67eed6e08e14306b70f135176 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 2 Mar 2016 02:50:15 +0100
Subject: [PATCH 0050/1225] [common] fix #943

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 17f84c3470..f22295cd7a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1129,7 +1129,7 @@ def version():
 
         elif o in ('-l', '--playlist'):
             playlist = True
-        elif o in ('--no-caption'):
+        elif o in ('--no-caption',):
             caption = False
         elif o in ('-n', '--no-merge'):
             merge = False

From a577c8553e6c8b410aa1ed4658ac232f20d56c64 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Mar 2016 03:55:51 +0100
Subject: [PATCH 0051/1225] [bilibili] hint for cookies

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 8a16222ab4..104f5b8ce3 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -112,6 +112,8 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
 
     title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
                    r'<h1[^>]*>([^<>]+)</h1>'], html)
+    if not title:
+        log.wtf('[Failed] Video does not exist. Try to login with --cookies.')
     title = unescape_html(title)
     title = escape_file_path(title)
 

From abf477f796684644becccc3efd04d7c3b77f6d9b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Mar 2016 04:00:44 +0100
Subject: [PATCH 0052/1225] [bilibili] hint for DNS failure

---
 src/you_get/extractors/bilibili.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 104f5b8ce3..0dca91e580 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -99,9 +99,12 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
 
     type_ = ''
     size = 0
-    for url in urls:
-        _, type_, temp = url_info(url)
-        size += temp or 0
+    try:
+        for url in urls:
+            _, type_, temp = url_info(url)
+            size += temp or 0
+    except:
+        log.wtf('[Failed] DNS not resolved. Please change your DNS server settings.')
 
     print_info(site_info, title, type_, size)
     if not info_only:

From fd1c63618af0a3bf7a49d6a80af1c7e81767afa1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Mar 2016 21:49:47 +0100
Subject: [PATCH 0053/1225] [letv] fix #947

---
 src/you_get/common.py                     |  3 ++-
 src/you_get/extractors/__init__.py        |  2 +-
 src/you_get/extractors/acfun.py           |  2 +-
 src/you_get/extractors/embed.py           |  2 +-
 src/you_get/extractors/{letv.py => le.py} | 10 ++++------
 5 files changed, 9 insertions(+), 10 deletions(-)
 rename src/you_get/extractors/{letv.py => le.py} (95%)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f22295cd7a..a92709f9c3 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -40,7 +40,8 @@
     'ku6'              : 'ku6',
     'kugou'            : 'kugou',
     'kuwo'             : 'kuwo',
-    'letv'             : 'letv',
+    'le'               : 'le',
+    'letv'             : 'le',
     'lizhi'            : 'lizhi',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index dfd45596bb..853165205c 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -31,7 +31,7 @@
 from .ku6 import *
 from .kugou import *
 from .kuwo import *
-from .letv import *
+from .le import *
 from .lizhi import *
 from .magisto import *
 from .metacafe import *
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index b120605687..e78d363627 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -4,7 +4,7 @@
 
 from ..common import *
 
-from .letv import letvcloud_download_by_vu
+from .le import letvcloud_download_by_vu
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_iid
diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index ef66b0df09..b594b9f48c 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -3,7 +3,7 @@
 from ..common import *
 
 from .iqiyi import iqiyi_download_by_vid
-from .letv import letvcloud_download_by_vu
+from .le import letvcloud_download_by_vu
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
diff --git a/src/you_get/extractors/letv.py b/src/you_get/extractors/le.py
similarity index 95%
rename from src/you_get/extractors/letv.py
rename to src/you_get/extractors/le.py
index 54da21f421..9de6c7a7fe 100644
--- a/src/you_get/extractors/letv.py
+++ b/src/you_get/extractors/le.py
@@ -130,14 +130,12 @@ def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
         letvcloud_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
         html = get_content(url)
-        #to get title
-        if re.match(r'http://www.letv.com/ptv/vplay/(\d+).html', url):
-            vid = match1(url, r'http://www.letv.com/ptv/vplay/(\d+).html')
-        else:
-            vid = match1(html, r'vid="(\d+)"')
+        vid = match1(url, r'http://www.letv.com/ptv/vplay/(\d+).html') or \
+            match1(url, r'http://www.le.com/ptv/vplay/(\d+).html') or \
+            match1(html, r'vid="(\d+)"')
         title = match1(html,r'name="irTitle" content="(.*?)"')
         letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
 
-site_info = "LeTV.com"
+site_info = "Le.com"
 download = letv_download
 download_playlist = playlist_not_supported('letv')

From 269829e77ac1ddbece0e57eefeb0a48873100bb7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Mar 2016 21:54:14 +0100
Subject: [PATCH 0054/1225] update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 7c1b211b5c..73e4f2faca 100644
--- a/README.md
+++ b/README.md
@@ -350,7 +350,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 酷6网    | <http://www.ku6.com/>          |✓| | |
 | 酷狗音乐 | <http://www.kugou.com/>        | | |✓|
 | 酷我音乐 | <http://www.kuwo.cn/>          | | |✓|
-| 乐视网   | <http://www.letv.com/>         |✓| | |
+| 乐视网   | <http://www.le.com/>           |✓| | |
 | 荔枝FM   | <http://www.lizhi.fm/>         | | |✓|
 | 秒拍     | <http://www.miaopai.com/>      |✓| | |
 | MioMio弹幕网 | <http://www.miomio.tv/>    |✓| | |

From 4173268a8e7b6b1a56715e21860aee570bb95f67 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Mar 2016 23:57:39 +0100
Subject: [PATCH 0055/1225] [755] support direct URLs (stat.7gogo.jp)

---
 src/you_get/extractors/nanagogo.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index 6f94043927..6382afc444 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -3,8 +3,13 @@
 __all__ = ['nanagogo_download']
 
 from ..common import *
+from .universal import *
 
 def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if re.match(r'https?://stat.7gogo.jp', url):
+        universal_download(url, output_dir, merge=merge, info_only=info_only)
+        return
+
     talk_id = r1(r'7gogo.jp/([^/]+)/', url)
     post_id = r1(r'7gogo.jp/[^/]+/(\d+)', url)
     title = '%s_%s' % (talk_id, post_id)

From 6df6a81e7d4d963ba11148a9bf043a2a11edd13a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 4 Mar 2016 14:46:14 +0100
Subject: [PATCH 0056/1225] [twitter] fix #948

---
 src/you_get/extractors/twitter.py | 39 +++++++++++++++----------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 058c827c37..91456491f3 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -41,26 +41,25 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                               output_dir=output_dir)
 
     except: # extract video
-        icards = r1(r'data-src="([^"]*)"', html)
-        if icards:
-            card = get_html("https://twitter.com" + icards)
-            data_player_config = r1(r'data-player-config="([^"]*)"', card)
-            if data_player_config is None:
-                vine_src = r1(r'<iframe src="([^"]*)"', card)
-                vine_download(vine_src, output_dir=output_dir, merge=merge, info_only=info_only)
-                return
-            data = json.loads(unescape_html(data_player_config))
-            if 'playlist' in data:
-                source = data['playlist'][0]['source']
-            else:
-                vmap = get_content(data['vmapUrl'])
-                source = r1(r'<!\[CDATA\[(.*)\]\]>', vmap)
-        else:
-            source = r1(r'<source video-src="([^"]*)"', html)
-            if not source:
-                vmap_url = r1(r'<meta name="twitter:amplify:vmap" content="([^"]+)"', html)
-                vmap = get_content(vmap_url)
-                source = r1(r'<!\[CDATA\[(.*)\]\]>', vmap)
+        # always use i/cards or videos url
+        if not re.match(r'https?://twitter.com/i/', url):
+            url = r1(r'<meta\s*property="og:video:url"\s*content="([^"]+)"', html)
+            html = get_content(url)
+
+        data_config = r1(r'data-config="([^"]*)"', html) or \
+            r1(r'data-player-config="([^"]*)"', html)
+        i = json.loads(unescape_html(data_config))
+        if 'video_url' in i:
+            source = i['video_url']
+            if not item_id: page_title = i['tweet_id']
+        elif 'playlist' in i:
+            source = i['playlist'][0]['source']
+            if not item_id: page_title = i['playlist'][0]['contentId']
+        elif 'vmap_url' in i:
+            vmap_url = i['vmap_url']
+            vmap = get_content(vmap_url)
+            source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
+            if not item_id: page_title = i['tweet_id']
 
         mime, ext, size = url_info(source)
 

From 35a5dac007a4b829cd55f8058f3186d66bc8cca6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 02:26:29 +0100
Subject: [PATCH 0057/1225] [infoq] add support

---
 src/you_get/common.py              |  2 ++
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/infoq.py    | 57 ++++++++++++++++++++++++++++++
 3 files changed, 60 insertions(+)
 create mode 100644 src/you_get/extractors/infoq.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a92709f9c3..30bafe5df1 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -28,6 +28,7 @@
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
     'in'               : 'alive',
+    'infoq'            : 'infoq',
     'instagram'        : 'instagram',
     'interest'         : 'interest',
     'iqilu'            : 'iqilu',
@@ -365,6 +366,7 @@ def url_info(url, faker = False, headers = {}):
         'image/jpeg': 'jpg',
         'image/png': 'png',
         'image/gif': 'gif',
+        'application/pdf': 'pdf',
     }
     if type in mapping:
         ext = mapping[type]
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 853165205c..b5817a3666 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -22,6 +22,7 @@
 from .heavymusic import *
 from .ifeng import *
 from .imgur import *
+from .infoq import *
 from .instagram import *
 from .interest import *
 from .iqilu import *
diff --git a/src/you_get/extractors/infoq.py b/src/you_get/extractors/infoq.py
new file mode 100644
index 0000000000..8214b47353
--- /dev/null
+++ b/src/you_get/extractors/infoq.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+import ssl
+
+class Infoq(VideoExtractor):
+    name = "InfoQ"
+
+    stream_types = [
+        {'id': 'video'},
+        {'id': 'audio'},
+        {'id': 'slides'}
+    ]
+
+    def prepare(self, **kwargs):
+        content = get_content(self.url)
+        self.title = match1(content, r'<title>([^<]+)</title>')
+        s = match1(content, r'P\.s\s*=\s*\'([^\']+)\'')
+        scp = match1(content, r'InfoQConstants\.scp\s*=\s*\'([^\']+)\'')
+        scs = match1(content, r'InfoQConstants\.scs\s*=\s*\'([^\']+)\'')
+        sck = match1(content, r'InfoQConstants\.sck\s*=\s*\'([^\']+)\'')
+
+        mp3 = match1(content, r'name="filename"\s*value="([^"]+\.mp3)"')
+        if mp3: mp3 = 'http://res.infoq.com/downloads/mp3downloads/%s' % mp3
+
+        pdf = match1(content, r'name="filename"\s*value="([^"]+\.pdf)"')
+        if pdf: pdf = 'http://res.infoq.com/downloads/pdfdownloads/%s' % pdf
+
+        # cookie handler
+        ssl_context = request.HTTPSHandler(
+            context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
+        cookie_handler = request.HTTPCookieProcessor()
+        opener = request.build_opener(ssl_context, cookie_handler)
+        opener.addheaders = [
+            ('Referer', self.url),
+            ('Cookie',
+             'CloudFront-Policy=%s;CloudFront-Signature=%s;CloudFront-Key-Pair-Id=%s' % (scp, scs, sck))
+        ]
+        request.install_opener(opener)
+
+        self.streams = {
+            'video'  : { 'url': s },
+            'audio'  : { 'url': mp3 },
+            'slides' : { 'url': pdf }
+        }
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+
+site = Infoq()
+download = site.download_by_url
+download_playlist = site.download_by_url

From c587af77417f36b1045d7fed28678d36665d5b05 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 02:26:56 +0100
Subject: [PATCH 0058/1225] update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 73e4f2faca..2591edbdf3 100644
--- a/README.md
+++ b/README.md
@@ -319,6 +319,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | JPopsuki TV | <http://www.jpopsuki.tv/>     |✓| | |
 | Internet Archive | <https://archive.org/>   |✓| | |
 | **Instagram** | <https://instagram.com/>    |✓|✓| |
+| InfoQ       | <http://www.infoq.com/presentations/> |✓| | |
 | Imgur       | <http://imgur.com/>           | |✓| |
 | Heavy Music Archive | <http://www.heavy-music.ru/> | | |✓|
 | **Google+** | <https://plus.google.com/>    |✓|✓| |

From 1a8f33271287bbb5a7483071bad17fabee9e12f6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 13:46:35 +0100
Subject: [PATCH 0059/1225] [iqiyi] strip Unicode character \u200b

---
 src/you_get/extractors/iqiyi.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 2700627d6b..a686f47901 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -135,6 +135,7 @@ def prepare(self, **kwargs):
             log.wtf("is your you-get up-to-date?")
 
         self.title = info["data"]["vi"]["vn"]
+        self.title = self.title.replace('\u200b', '')
 
         # data.vp = json.data.vp
         #  data.vi = json.data.vi

From cd5cfc775f3ca52544b98f0ce343319791f3eca2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 16:18:14 +0100
Subject: [PATCH 0060/1225] [common] maybe_print() ignores any non-printable
 characters (which used to cause an exception, which is not intuitive)

---
 src/you_get/common.py    |  8 ++++++--
 src/you_get/extractor.py | 10 +++++-----
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 30bafe5df1..0c5fa86fb0 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -126,6 +126,10 @@
 else:
     default_encoding = locale.getpreferredencoding().lower()
 
+def maybe_print(s):
+    try: print(s)
+    except: pass
+
 def tr(s):
     if default_encoding == 'utf-8':
         return s
@@ -953,8 +957,8 @@ def print_info(site_info, title, type, size):
     else:
         type_info = "Unknown type (%s)" % type
 
-    print("Site:      ", site_info)
-    print("Title:     ", unescape_html(tr(title)))
+    maybe_print("Site:      ", site_info)
+    maybe_print("Title:     ", unescape_html(tr(title)))
     print("Type:      ", type_info)
     print("Size:      ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
     print()
diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 5ca7fe67c2..23bb2ac3f2 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from .common import match1, download_urls, get_filename, parse_host, set_proxy, unset_proxy
+from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy
 from .util import log
 from . import json_output
 import os
@@ -111,14 +111,14 @@ def p_i(self, stream_id):
         else:
             stream = self.dash_streams[stream_id]
 
-        print("    - title:         %s" % self.title)
+        maybe_print("    - title:         %s" % self.title)
         print("       size:         %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
         print("        url:         %s" % self.url)
         print()
 
     def p(self, stream_id=None):
-        print("site:                %s" % self.__class__.name)
-        print("title:               %s" % self.title)
+        maybe_print("site:                %s" % self.__class__.name)
+        maybe_print("title:               %s" % self.title)
         if stream_id:
             # Print the stream
             print("stream:")
@@ -151,7 +151,7 @@ def p(self, stream_id=None):
                 print("      download-url:  {}\n".format(i['url']))
 
     def p_playlist(self, stream_id=None):
-        print("site:                %s" % self.__class__.name)
+        maybe_print("site:                %s" % self.__class__.name)
         print("playlist:            %s" % self.title)
         print("videos:")
 

From 221fda25b306f56263f488fc4058334c114bf643 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 16:58:57 +0100
Subject: [PATCH 0061/1225] [common] maybe_print(): fix positional arguments

---
 src/you_get/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 0c5fa86fb0..a60d9b0463 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -126,8 +126,8 @@
 else:
     default_encoding = locale.getpreferredencoding().lower()
 
-def maybe_print(s):
-    try: print(s)
+def maybe_print(*s):
+    try: print(*s)
     except: pass
 
 def tr(s):

From 511098e71f1aed950cb03526eb82279d63f94b3f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 17:48:09 +0100
Subject: [PATCH 0062/1225] [iqiyi] add download_playlist_by_url(), fix #940

---
 src/you_get/extractors/iqiyi.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index a686f47901..68823deb9d 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -112,7 +112,14 @@ def getVMS(self):
                 "&authkey="+hashlib.new('md5',bytes(hashlib.new('md5', b'').hexdigest()+str(tm)+tvid,'utf-8')).hexdigest()
         return json.loads(get_content(vmsreq))
 
+    def download_playlist_by_url(self, url, **kwargs):
+        self.url = url
 
+        video_page = get_content(url)
+        videos = set(re.findall(r'<a href="(http://www\.iqiyi\.com/v_[^"]+)"', video_page))
+
+        for video in videos:
+            self.__class__().download_by_url(video, **kwargs)
 
     def prepare(self, **kwargs):
         assert self.url or self.vid
@@ -127,8 +134,12 @@ def prepare(self, **kwargs):
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
 
-        self.gen_uid=uuid4().hex
-        info = self.getVMS()
+        self.gen_uid = uuid4().hex
+        try:
+            info = self.getVMS()
+        except:
+            self.download_playlist_by_url(self.url, **kwargs)
+            exit(0)
 
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
@@ -201,4 +212,4 @@ def extract(self, **kwargs):
 site = Iqiyi()
 download = site.download_by_url
 iqiyi_download_by_vid = site.download_by_vid
-download_playlist = playlist_not_supported('iqiyi')
+download_playlist = site.download_playlist_by_url

From 6c2c29f72468b79bbf8543412b503d7ba44ec883 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 19:43:17 +0100
Subject: [PATCH 0063/1225] [youtube] fix for age-restricted videos, which do
 not contain ytplayer.config (or html5player) on web page

---
 src/you_get/extractors/youtube.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 70a5f1cb22..2cfd1ec8ae 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -152,8 +152,11 @@ def prepare(self, **kwargs):
 
                 # Parse video page (for DASH)
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-                ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-                self.html5player = 'https:' + ytplayer_config['assets']['js']
+                try:
+                    ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
+                    self.html5player = 'https:' + ytplayer_config['assets']['js']
+                except:
+                    self.html5player = None
 
             else:
                 # Parse video page instead
@@ -294,6 +297,7 @@ def prepare(self, **kwargs):
                         }
         except:
             # VEVO
+            if not self.html5player: return
             self.js = get_content(self.html5player)
             if 'adaptive_fmts' in ytplayer_config['args']:
                 streams = [dict([(i.split('=')[0],

From bf9910ad9796866ff825edea147ba5d7b199f86b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Mar 2016 20:17:43 +0100
Subject: [PATCH 0064/1225] version 0.4.324

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 68443fba45..d4fd303356 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.306'
+__version__ = '0.4.324'

From ad6fcf2557f80069780425d5cdf5a9b9fba2bb34 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 6 Mar 2016 03:44:49 +0100
Subject: [PATCH 0065/1225] [infoq] fix #957

---
 src/you_get/extractors/infoq.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/infoq.py b/src/you_get/extractors/infoq.py
index 8214b47353..cf8b59e0b1 100644
--- a/src/you_get/extractors/infoq.py
+++ b/src/you_get/extractors/infoq.py
@@ -40,11 +40,9 @@ def prepare(self, **kwargs):
         ]
         request.install_opener(opener)
 
-        self.streams = {
-            'video'  : { 'url': s },
-            'audio'  : { 'url': mp3 },
-            'slides' : { 'url': pdf }
-        }
+        if s: self.streams['video'] = {'url': s }
+        if mp3: self.streams['audio'] = { 'url': mp3 }
+        if pdf: self.streams['slides'] = { 'url': pdf }
 
     def extract(self, **kwargs):
         for i in self.streams:

From ac19e9656619c74b011eb696e332914d27c05f0d Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sun, 6 Mar 2016 20:20:15 -0500
Subject: [PATCH 0066/1225] [.github] Ask for reading FAQ

---
 .github/ISSUE_TEMPLATE.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 5cf8212b3a..8556750719 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -2,7 +2,8 @@ Please make sure these boxes are checked before submitting your issue – thank
 
 - [ ] You can actually watch the video in your browser or mobile application, but not download them with `you-get`.
 - [ ] Your `you-get` is up-to-date.
-- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or <https://github.com/soimort/you-get/wiki/Known-Bugs>. If so, please add your comments under the existing issue.
+- [ ] I have read <https://github.com/soimort/you-get/wiki/FAQ> and tried to do so.
+- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or  <https://github.com/soimort/you-get/wiki/Known-Bugs>. If so, please add your comments under the existing issue.
 - [ ] The issue (or question) is really about `you-get`, not about some other code or project.
 
 Run the command with the `--debug` option, and paste the full output inside the fences:
@@ -22,7 +23,8 @@ If there's anything else you would like to say (e.g. in case your issue is not a
 
 - [ ] 你可以在浏览器或移动端中观看视频，但不能使用`you-get`下载.
 - [ ] 您的`you-get`为最新版.
-- [ ] 您的问题没有在<https://github.com/soimort/you-get/issues> 或 <https://github.com/soimort/you-get/wiki/Known-Bugs> 报告，否则请在原有issue下报告.
+- [ ] 我已经阅读并按 <https://github.com/soimort/you-get/wiki/FAQ> 中的指引进行了操作.
+- [ ] 您的问题没有在<https://github.com/soimort/you-get/issues> , <https://github.com/soimort/you-get/wiki/FAQ> 或 <https://github.com/soimort/you-get/wiki/Known-Bugs> 报告，否则请在原有issue下报告.
 - [ ] 本问题确实关于`you-get`, 而不是其他项目.
 
 请使用`--debug`运行，并将输出粘贴在下面:

From 0ac3bdd5b3139976502a79ef2cf684c92420e9cb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 10 Mar 2016 19:25:18 +0100
Subject: [PATCH 0067/1225] [bilibili] change to generic appkey, fix #973

---
 src/you_get/extractors/bilibili.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 0dca91e580..7522a6e5c7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -11,17 +11,7 @@
 import hashlib
 import re
 
-# API key provided by cnbeining
-appkey='85eb6835b0a1034e';
-secretkey = '2ad42749773c441109bdc0191257a664'
-client = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Charset': 'UTF-8,*;q=0.5',
-    'Accept-Encoding': 'gzip,deflate,sdch',
-    'Accept-Language': 'en-US,en;q=0.8',
-    #'User-Agent': 'Biligrab /0.8 (cnbeining@gmail.com)'
-    'User-Agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.94 Safari/537.36"
-}
+appkey='8e9fc618fbd41e28'
 
 def get_srt_xml(id):
     url = 'http://comment.bilibili.com/%s.xml' % id
@@ -72,12 +62,11 @@ def parse_cid_playurl(xml):
 def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
     urls = []
     for cid in cids:
-        sign_this = hashlib.md5(bytes('appkey=' + appkey + '&cid=' + cid + secretkey, 'utf-8')).hexdigest()
-        url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid + '&sign=' + sign_this
+        url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
         urls += [i
                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                for i in parse_cid_playurl(get_content(url, headers=client))]
+                for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -90,12 +79,11 @@ def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
 def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    sign_this = hashlib.md5(bytes('appkey=' + appkey + '&cid=' + cid + secretkey, 'utf-8')).hexdigest()
-    url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid + '&sign=' + sign_this
+    url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
     urls = [i
             if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
             else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-            for i in parse_cid_playurl(get_content(url, headers=client))]
+            for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -103,7 +91,7 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
         for url in urls:
             _, type_, temp = url_info(url)
             size += temp or 0
-    except:
+    except error.URLError:
         log.wtf('[Failed] DNS not resolved. Please change your DNS server settings.')
 
     print_info(site_info, title, type_, size)

From 6590bb33cd52c89df750b4d67d4b61e853709ceb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 10 Mar 2016 19:30:15 +0100
Subject: [PATCH 0068/1225] [iqiyi] close #972

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 68823deb9d..e3cca88dad 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -45,7 +45,7 @@
 
 '''
 def mix(tvid):
-    salt = '6ab6d0280511493ba85594779759d4ed'
+    salt = '8ed797d224d043e7ac23d95b70227d32'
     tm = str(randint(2000,4000))
     sc = hashlib.new('md5', bytes(salt + tm + tvid, 'utf-8')).hexdigest()
     return tm, sc, 'eknas'

From 225f18d347a8e19015a13b22a40d033b0a32a7c9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 10 Mar 2016 22:48:35 +0100
Subject: [PATCH 0069/1225] [common] catch UnicodeEncodeError and instruct
 users to change the locale

---
 src/you_get/common.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a60d9b0463..a76dc5b187 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1183,6 +1183,14 @@ def version():
             raise
         else:
             sys.exit(1)
+    except UnicodeEncodeError:
+        log.e('[error] oops, the current environment does not seem to support Unicode.')
+        log.e('please set it to a UTF-8-aware locale first,')
+        log.e('so as to save the video (with some Unicode characters) correctly.')
+        log.e('you can do it like this:')
+        log.e('    (Windows)    % chcp 65001 ')
+        log.e('    (Linux)      $ LC_CTYPE=en_US.UTF-8')
+        sys.exit(1)
     except Exception:
         if not traceback:
             log.e('[error] oops, something went wrong.')

From bf495b1bed4966aeda54c4294b9023c4d97e3331 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 11 Mar 2016 00:00:00 +0100
Subject: [PATCH 0070/1225] version 0.4.330

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index d4fd303356..99aeb880e0 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.324'
+__version__ = '0.4.330'

From a2308ad2a1b918e690573586746ecfbaeb2158ca Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Sun, 6 Sep 2015 18:20:03 +0800
Subject: [PATCH 0071/1225] reset url and vid when download in serial

when use xxx_download_by_url(url1/url2)
vid will not reset, only first url will download

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractor.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 23bb2ac3f2..9bc950fc28 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -33,6 +33,7 @@ def __init__(self, *args):
 
     def download_by_url(self, url, **kwargs):
         self.url = url
+        self.vid= None
 
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             set_proxy(parse_host(kwargs['extractor_proxy']))
@@ -50,6 +51,7 @@ def download_by_url(self, url, **kwargs):
         self.download(**kwargs)
 
     def download_by_vid(self, vid, **kwargs):
+        self.url = None
         self.vid = vid
 
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:

From cc1840f187783a27c4514e78dcdb3c18e67a76f0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 13 Mar 2016 19:41:34 +0100
Subject: [PATCH 0072/1225] [extractor] use maybe_print for video-profile (fix
 #982)

---
 src/you_get/extractor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 23bb2ac3f2..8ebc735182 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -90,7 +90,7 @@ def p_stream(self, stream_id):
             print("      container:     %s" % stream['container'])
 
         if 'video_profile' in stream:
-            print("      video-profile: %s" % stream['video_profile'])
+            maybe_print("      video-profile: %s" % stream['video_profile'])
 
         if 'quality' in stream:
             print("      quality:       %s" % stream['quality'])

From 80ea797cb688a8022715485255239b9b51d1dd04 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 15 Mar 2016 05:11:34 +0100
Subject: [PATCH 0073/1225] [bilibili] add support for live.bilibili.com (close
 #986)

---
 src/you_get/extractors/bilibili.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 7522a6e5c7..cd9186029d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -98,15 +98,25 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
+    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
+    urls = parse_cid_playurl(get_content(api_url))
+
+    for url in urls:
+        _, type_, _ = url_info(url)
+        size = 0
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
     title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
                    r'<h1[^>]*>([^<>]+)</h1>'], html)
-    if not title:
-        log.wtf('[Failed] Video does not exist. Try to login with --cookies.')
-    title = unescape_html(title)
-    title = escape_file_path(title)
+    if title:
+        title = unescape_html(title)
+        title = escape_file_path(title)
 
     flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
     assert flashvars
@@ -114,7 +124,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     t, cid = flashvars.split('=', 1)
     cid = cid.split('&')[0]
     if t == 'cid':
-        if 'playlist' in kwargs and kwargs['playlist']:
+        if re.match(r'https?://live\.bilibili\.com/', url):
+            title = r1(r'<title>([^<>]+)</title>', html)
+            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+        elif 'playlist' in kwargs and kwargs['playlist']:
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)

From 1848dbe7d7e64406466afbb7895684989a471b69 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Tue, 15 Mar 2016 01:46:37 -0400
Subject: [PATCH 0074/1225] [CKPlayer]Extract by XML

To be used for convenience. Downloader not fully tested!
---
 src/you_get/extractors/ckplayer.py | 78 ++++++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 src/you_get/extractors/ckplayer.py

diff --git a/src/you_get/extractors/ckplayer.py b/src/you_get/extractors/ckplayer.py
new file mode 100644
index 0000000000..2cdc352a4a
--- /dev/null
+++ b/src/you_get/extractors/ckplayer.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+#coding:utf-8
+# Author:  Beining --<i@cnbeining.com>
+# Purpose: A general extractor for CKPlayer
+# Created: 03/15/2016
+
+__all__ = ['ckplayer_download']
+
+from xml.etree import cElementTree as ET
+from copy import copy
+from ..common import *
+
+#----------------------------------------------------------------------
+def get_info_by_xml(ckinfo):
+    """str->dict
+    Information for CKPlayer API content."""
+    e = ET.XML(ckinfo)
+    video_dict = {'title': '',
+                  #'duration': 0,
+                  'links': [],
+                  'size': 0,
+                  'flashvars': '',}
+    if '_text' in dictify(e)['ckplayer']['info'][0]['title'][0]:  #title
+        video_dict['title'] = dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip()
+
+    #if dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip():  #duration
+        #video_dict['title'] = dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip()
+
+    if '_text' in dictify(e)['ckplayer']['video'][0]['size'][0]:  #size exists for 1 piece
+        video_dict['size'] = sum([int(i['size'][0]['_text']) for i in dictify(e)['ckplayer']['video']])
+
+    if '_text' in dictify(e)['ckplayer']['video'][0]['file'][0]:  #link exist
+        video_dict['links'] = [i['file'][0]['_text'].strip() for i in dictify(e)['ckplayer']['video']]
+
+    if '_text' in dictify(e)['ckplayer']['flashvars'][0]:
+        video_dict['flashvars'] = dictify(e)['ckplayer']['flashvars'][0]['_text'].strip()
+
+    return video_dict
+
+#helper
+#https://stackoverflow.com/questions/2148119/how-to-convert-an-xml-string-to-a-dictionary-in-python
+def dictify(r,root=True):
+    if root:
+        return {r.tag : dictify(r, False)}
+    d=copy(r.attrib)
+    if r.text:
+        d["_text"]=r.text
+    for x in r.findall("./*"):
+        if x.tag not in d:
+            d[x.tag]=[]
+        d[x.tag].append(dictify(x,False))
+    return d
+
+def ckplayer_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    #Info XML
+    ckinfo = get_content(url)
+    video_info = get_info_by_xml(ckinfo)
+    
+    title = ''
+    type_ = ''
+    size = 0
+    
+    if len(video_info['links']) > 0:  #has link
+        type_, _ext, size = url_info(video_info['links'][0])  #use 1st to determine type, ext
+    
+    if 'size' in video_info:
+        size = int(video_info['size'])
+    else:
+        for i in video_info['links'][1:]:  #save 1st one
+            size += url_info(i)[2]
+    
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_urls(video_info['links'], title, ext, size, output_dir=output_dir, merge=merge)
+
+site_info = "CKPlayer General"
+download = ckplayer_download
+download_playlist = playlist_not_supported('ckplayer')

From b35a0bd9f92a6298f58bb5281ba2bfd541d04e1e Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Tue, 15 Mar 2016 02:41:23 -0400
Subject: [PATCH 0075/1225] [CKPlayer]Refractionation to provide better support
 of headers, etc

---
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/ckplayer.py | 32 ++++++++++++++++++++++++------
 2 files changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index b5817a3666..15c0c7222e 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -7,6 +7,7 @@
 from .bandcamp import *
 from .bilibili import *
 from .cbs import *
+from .ckplayer import *
 from .cntv import *
 from .dailymotion import *
 from .dilidili import *
diff --git a/src/you_get/extractors/ckplayer.py b/src/you_get/extractors/ckplayer.py
index 2cdc352a4a..09e95557a6 100644
--- a/src/you_get/extractors/ckplayer.py
+++ b/src/you_get/extractors/ckplayer.py
@@ -11,7 +11,7 @@
 from ..common import *
 
 #----------------------------------------------------------------------
-def get_info_by_xml(ckinfo):
+def ckplayer_get_info_by_xml(ckinfo):
     """str->dict
     Information for CKPlayer API content."""
     e = ET.XML(ckinfo)
@@ -37,6 +37,7 @@ def get_info_by_xml(ckinfo):
 
     return video_dict
 
+#----------------------------------------------------------------------
 #helper
 #https://stackoverflow.com/questions/2148119/how-to-convert-an-xml-string-to-a-dictionary-in-python
 def dictify(r,root=True):
@@ -51,12 +52,15 @@ def dictify(r,root=True):
         d[x.tag].append(dictify(x,False))
     return d
 
-def ckplayer_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+#----------------------------------------------------------------------
+def ckplayer_download_by_xml(ckinfo, output_dir = '.', merge = False, info_only = False, **kwargs):
     #Info XML
-    ckinfo = get_content(url)
-    video_info = get_info_by_xml(ckinfo)
+    video_info = ckplayer_get_info_by_xml(ckinfo)
     
-    title = ''
+    try:
+        title = kwargs['title']
+    except:
+        title = ''
     type_ = ''
     size = 0
     
@@ -71,7 +75,23 @@ def ckplayer_download(url, output_dir = '.', merge = False, info_only = False, *
     
     print_info(site_info, title, type_, size)
     if not info_only:
-        download_urls(video_info['links'], title, ext, size, output_dir=output_dir, merge=merge)
+        download_urls(video_info['links'], title, _ext, size, output_dir=output_dir, merge=merge)
+
+#----------------------------------------------------------------------
+def ckplayer_download(url, output_dir = '.', merge = False, info_only = False, is_xml = True, **kwargs):
+    if is_xml:  #URL is XML URL
+        try:
+            title = kwargs['title']
+        except:
+            title = ''
+        try:
+            headers = kwargs['headers']  #headers provided
+            ckinfo = get_content(url, headers = headers)
+        except NameError:
+            ckinfo = get_content(url)
+        
+        ckplayer_download_by_xml(ckinfo, output_dir, merge, 
+                                info_only, title = title)
 
 site_info = "CKPlayer General"
 download = ckplayer_download

From 1746adab2251904daf4bbd6bc0010ee23015c542 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Tue, 15 Mar 2016 03:00:07 -0400
Subject: [PATCH 0076/1225] [dilidili]Fix #987, close #986

---
 src/you_get/extractors/dilidili.py | 58 ++++++++++++++++++++----------
 1 file changed, 39 insertions(+), 19 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/dilidili.py

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
old mode 100644
new mode 100755
index 1c5340a63b..c68c17a692
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -3,12 +3,26 @@
 __all__ = ['dilidili_download']
 
 from ..common import *
+from .ckplayer import ckplayer_download
+
+headers = {
+    'DNT': '1',
+    'Accept-Encoding': 'gzip, deflate, sdch, br',
+    'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Cache-Control': 'max-age=0',
+    'Referer': 'http://www.dilidili.com/',
+    'Connection': 'keep-alive',
+    'Save-Data': 'on',
+}
 
 #----------------------------------------------------------------------
-def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign):
+def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
     """->list"""
-    parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign)
-    html = get_html(parse_url)
+    parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
+    html = get_content(parse_url, headers=headers)
     
     info = re.search(r'(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})', html).groups()
     info = [i.strip('{}').split('->') for i in info]
@@ -19,46 +33,52 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign):
         stream_types.append({'id': str(i[1][-1]), 'container': 'mp4', 'video_profile': i[0]})
     return stream_types
 
-#----------------------------------------------------------------------
-def dilidili_parser_data_to_download_url(typ ,vid ,hd2 ,sign):
-    """->str"""
-    parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign)
-    html = get_html(parse_url)
+##----------------------------------------------------------------------
+#def dilidili_parser_data_to_download_url(typ ,vid ,hd2 ,sign, tmsign, ulk):
+    #"""Not used for now"""
+    #parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
+    #html = get_content(parse_url, headers=headers)
     
-    return match1(html, r'<file><!\[CDATA\[(.+)\]\]></file>')
+    #return match1(html, r'<file><!\[CDATA\[(.+)\]\]></file>')
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     if re.match(r'http://www.dilidili.com/watch/\w+', url):
-        html = get_html(url)
+        html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         
         # player loaded via internal iframe
         frame_url = re.search(r'<iframe (.+)src="(.+)\" f(.+)</iframe>', html).group(2)
+        #print(frame_url)
+        
         #https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
-        html = get_html(frame_url)
+        html = get_content(frame_url, headers=headers)
         
         match = re.search(r'(.+?)var video =(.+?);', html)
         vid = match1(html, r'var vid="(.+)"')
         hd2 = match1(html, r'var hd2="(.+)"')
         typ = match1(html, r'var typ="(.+)"')
         sign = match1(html, r'var sign="(.+)"')
+        tmsign = match1(html, r'tmsign=([A-Za-z0-9]+)')
+        ulk =  match1(html, r'var ulk="(.+)"')
         
         # here s the parser...
-        stream_types = dilidili_parser_data_to_stream_types(typ, vid, hd2, sign)
+        stream_types = dilidili_parser_data_to_stream_types(typ, vid, hd2, sign, tmsign, ulk)
         
         #get best
         best_id = max([i['id'] for i in stream_types])
         
-        url = dilidili_parser_data_to_download_url(typ, vid, best_id, sign)
+        parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = best_id, sign = sign, tmsign = tmsign, ulk = ulk)
+        
+        ckplayer_download(parse_url, output_dir, merge, info_only, is_xml = True, title = title, headers = headers)
 
-        type_ = ''
-        size = 0
+        #type_ = ''
+        #size = 0
 
-        type_, ext, size = url_info(url)
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
+        #type_, ext, size = url_info(url)
+        #print_info(site_info, title, type_, size)
+        #if not info_only:
+            #download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
 site_info = "dilidili"
 download = dilidili_download

From 343c410973b6f48f457daa579b07b9ee370da5e6 Mon Sep 17 00:00:00 2001
From: wenLiangcan <boxeed@gmail.com>
Date: Fri, 11 Mar 2016 17:57:47 +0800
Subject: [PATCH 0077/1225] Add huaban.com support.

---
 .gitignore                         |   2 +
 README.md                          |   1 +
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/huaban.py   | 130 +++++++++++++++++++++++++++++
 5 files changed, 135 insertions(+)
 create mode 100644 src/you_get/extractors/huaban.py

diff --git a/.gitignore b/.gitignore
index 354bb10951..d22d3afe3a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,3 +79,5 @@ _*
 *.ts
 *.webm
 *.xml
+/.env
+/.idea
diff --git a/README.md b/README.md
index 2591edbdf3..abdf39e531 100644
--- a/README.md
+++ b/README.md
@@ -371,6 +371,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
+| 花瓣     | <http://huaban.com/>           | |✓| |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index a76dc5b187..f15481a388 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -24,6 +24,7 @@
     'fun'              : 'funshion',
     'google'           : 'google',
     'heavy-music'      : 'heavymusic',
+    'huaban'           : 'huaban',
     'iask'             : 'sina',
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 15c0c7222e..5af9cdd360 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -21,6 +21,7 @@
 from .funshion import *
 from .google import *
 from .heavymusic import *
+from .huaban import *
 from .ifeng import *
 from .imgur import *
 from .infoq import *
diff --git a/src/you_get/extractors/huaban.py b/src/you_get/extractors/huaban.py
new file mode 100644
index 0000000000..a011ae35b4
--- /dev/null
+++ b/src/you_get/extractors/huaban.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+
+import json
+import os
+import re
+import traceback
+import urllib.parse as urlparse
+
+from ..common import *
+
+__all__ = ['huaban_download']
+
+site_info = '花瓣 (Huaban)'
+
+LIMIT = 100
+
+
+class EnhancedPiecesProgressBar(PiecesProgressBar):
+    BAR_LEN = 40
+
+    def update(self):
+        self.displayed = True
+        bar = '{0:>5}%[{1}] {2}/{3}'.format(
+            '', '=' * self.done_bar + '-' * self.todo_bar,
+            self.current_piece, self.total_pieces)
+        sys.stdout.write('\r' + bar)
+        sys.stdout.flush()
+
+    @property
+    def done_bar(self):
+        return self.BAR_LEN // self.total_pieces * self.current_piece
+
+    @property
+    def todo_bar(self):
+        return self.BAR_LEN - self.done_bar
+
+
+class Board:
+    def __init__(self, title, pins):
+        self.title = title
+        self.pins = pins
+        self.pin_count = len(pins)
+
+
+class Pin:
+    host = 'http://img.hb.aicdn.com/'
+
+    def __init__(self, pin_json):
+        img_file = pin_json['file']
+        self.key = img_file['key']
+        self.url = urlparse.urljoin(self.host, self.key)
+        self.ext = img_file['type'].split('/')[-1]
+
+
+def construct_url(url, **params):
+    param_str = urlparse.urlencode(params)
+    return url + '?' + param_str
+
+
+def extract_json_data(url, **params):
+    url = construct_url(url, **params)
+    html = get_content(url, headers=fake_headers)
+    json_string = match1(html, r'app.page\["board"\] = (.*?});')
+    json_data = json.loads(json_string)
+    return json_data
+
+
+def extract_board_data(url):
+    json_data = extract_json_data(url, limit=LIMIT)
+    pin_list = json_data['pins']
+    title = json_data['title']
+    pin_count = json_data['pin_count']
+    pin_count -= len(pin_list)
+
+    while pin_count > 0:
+        json_data = extract_json_data(url, max=pin_list[-1]['pin_id'],
+                                      limit=LIMIT)
+        pins = json_data['pins']
+        pin_list += pins
+        pin_count -= len(pins)
+
+    return Board(title, list(map(Pin, pin_list)))
+
+
+def get_num_len(num):
+    return len(str(num))
+
+
+def huaban_download_board(url, output_dir, **kwargs):
+    board = extract_board_data(url)
+    output_dir = os.path.join(output_dir, board.title)
+    bar = EnhancedPiecesProgressBar(float('Inf'), board.pin_count)
+
+    print("Site:      ", site_info)
+    print("Title:     ", board.title)
+    print()
+
+    if dry_run:
+        urls = '\n'.join(map(lambda p: p.url, board.pins))
+        print('Real URLs:\n{}'.format(urls))
+        return
+
+    print('Downloading {} images in {} ...'.format(board.pin_count,
+                                                   board.title))
+    try:
+        bar.update()
+        name_len = get_num_len(board.pin_count)
+        for i, pin in enumerate(board.pins):
+            filename = '{0}[{1}].{2}'.format(board.title,
+                                             str(i).zfill(name_len), pin.ext)
+            filepath = os.path.join(output_dir, filename)
+            bar.update_piece(i + 1)
+            url_save(pin.url, filepath, bar, is_part=True, faker=True)
+        bar.done()
+    except KeyboardInterrupt:
+        pass
+    except:
+        traceback.print_exception(*sys.exc_info())
+
+
+def huaban_download(url, output_dir='.', **kwargs):
+    if re.match(r'http://huaban\.com/boards/\d+/', url):
+        huaban_download_board(url, output_dir, **kwargs)
+    else:
+        print('Only board (画板) pages are supported currently')
+        print('ex: http://huaban.com/boards/12345678/')
+
+
+download = huaban_download
+download_playlist = playlist_not_supported("huaban")

From 911794a3725c5bdda65b02470f7845b33587ddbe Mon Sep 17 00:00:00 2001
From: wenLiangcan <boxeed@gmail.com>
Date: Sat, 12 Mar 2016 09:52:53 +0800
Subject: [PATCH 0078/1225] Use pin id as output filename.

---
 src/you_get/extractors/huaban.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/huaban.py b/src/you_get/extractors/huaban.py
index a011ae35b4..efb082c1ce 100644
--- a/src/you_get/extractors/huaban.py
+++ b/src/you_get/extractors/huaban.py
@@ -47,8 +47,8 @@ class Pin:
 
     def __init__(self, pin_json):
         img_file = pin_json['file']
-        self.key = img_file['key']
-        self.url = urlparse.urljoin(self.host, self.key)
+        self.id = pin_json['pin_id']
+        self.url = urlparse.urljoin(self.host, img_file['key'])
         self.ext = img_file['type'].split('/')[-1]
 
 
@@ -106,8 +106,7 @@ def huaban_download_board(url, output_dir, **kwargs):
         bar.update()
         name_len = get_num_len(board.pin_count)
         for i, pin in enumerate(board.pins):
-            filename = '{0}[{1}].{2}'.format(board.title,
-                                             str(i).zfill(name_len), pin.ext)
+            filename = '{0}.{1}'.format(pin.id, pin.ext)
             filepath = os.path.join(output_dir, filename)
             bar.update_piece(i + 1)
             url_save(pin.url, filepath, bar, is_part=True, faker=True)

From ce10df775cf3ee8a438ab93055a9a5c70fa6fc42 Mon Sep 17 00:00:00 2001
From: wenLiangcan <boxeed@gmail.com>
Date: Sat, 12 Mar 2016 09:54:50 +0800
Subject: [PATCH 0079/1225] Fix progressbar animation.

---
 src/you_get/extractors/huaban.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/huaban.py b/src/you_get/extractors/huaban.py
index efb082c1ce..a8283a7257 100644
--- a/src/you_get/extractors/huaban.py
+++ b/src/you_get/extractors/huaban.py
@@ -3,6 +3,7 @@
 import json
 import os
 import re
+import math
 import traceback
 import urllib.parse as urlparse
 
@@ -28,7 +29,7 @@ def update(self):
 
     @property
     def done_bar(self):
-        return self.BAR_LEN // self.total_pieces * self.current_piece
+        return math.ceil(self.BAR_LEN / self.total_pieces * self.current_piece)
 
     @property
     def todo_bar(self):
@@ -82,10 +83,6 @@ def extract_board_data(url):
     return Board(title, list(map(Pin, pin_list)))
 
 
-def get_num_len(num):
-    return len(str(num))
-
-
 def huaban_download_board(url, output_dir, **kwargs):
     board = extract_board_data(url)
     output_dir = os.path.join(output_dir, board.title)
@@ -104,7 +101,6 @@ def huaban_download_board(url, output_dir, **kwargs):
                                                    board.title))
     try:
         bar.update()
-        name_len = get_num_len(board.pin_count)
         for i, pin in enumerate(board.pins):
             filename = '{0}.{1}'.format(pin.id, pin.ext)
             filepath = os.path.join(output_dir, filename)

From 052a6410430f024cfbc78cfbb08ba9fb52429d6f Mon Sep 17 00:00:00 2001
From: wenLiangcan <boxeed@gmail.com>
Date: Tue, 15 Mar 2016 12:05:02 +0800
Subject: [PATCH 0080/1225] [Huaban] Refactoring.

Remove customized file handling and logging code and make use of
`downalod_urls()` and `print_info()`.
---
 src/you_get/extractors/huaban.py | 52 ++++----------------------------
 1 file changed, 6 insertions(+), 46 deletions(-)

diff --git a/src/you_get/extractors/huaban.py b/src/you_get/extractors/huaban.py
index a8283a7257..8acf938bfa 100644
--- a/src/you_get/extractors/huaban.py
+++ b/src/you_get/extractors/huaban.py
@@ -16,26 +16,6 @@
 LIMIT = 100
 
 
-class EnhancedPiecesProgressBar(PiecesProgressBar):
-    BAR_LEN = 40
-
-    def update(self):
-        self.displayed = True
-        bar = '{0:>5}%[{1}] {2}/{3}'.format(
-            '', '=' * self.done_bar + '-' * self.todo_bar,
-            self.current_piece, self.total_pieces)
-        sys.stdout.write('\r' + bar)
-        sys.stdout.flush()
-
-    @property
-    def done_bar(self):
-        return math.ceil(self.BAR_LEN / self.total_pieces * self.current_piece)
-
-    @property
-    def todo_bar(self):
-        return self.BAR_LEN - self.done_bar
-
-
 class Board:
     def __init__(self, title, pins):
         self.title = title
@@ -48,7 +28,7 @@ class Pin:
 
     def __init__(self, pin_json):
         img_file = pin_json['file']
-        self.id = pin_json['pin_id']
+        self.id = str(pin_json['pin_id'])
         self.url = urlparse.urljoin(self.host, img_file['key'])
         self.ext = img_file['type'].split('/')[-1]
 
@@ -84,33 +64,13 @@ def extract_board_data(url):
 
 
 def huaban_download_board(url, output_dir, **kwargs):
+    kwargs['merge'] = False
     board = extract_board_data(url)
     output_dir = os.path.join(output_dir, board.title)
-    bar = EnhancedPiecesProgressBar(float('Inf'), board.pin_count)
-
-    print("Site:      ", site_info)
-    print("Title:     ", board.title)
-    print()
-
-    if dry_run:
-        urls = '\n'.join(map(lambda p: p.url, board.pins))
-        print('Real URLs:\n{}'.format(urls))
-        return
-
-    print('Downloading {} images in {} ...'.format(board.pin_count,
-                                                   board.title))
-    try:
-        bar.update()
-        for i, pin in enumerate(board.pins):
-            filename = '{0}.{1}'.format(pin.id, pin.ext)
-            filepath = os.path.join(output_dir, filename)
-            bar.update_piece(i + 1)
-            url_save(pin.url, filepath, bar, is_part=True, faker=True)
-        bar.done()
-    except KeyboardInterrupt:
-        pass
-    except:
-        traceback.print_exception(*sys.exc_info())
+    print_info(site_info, board.title, 'jpg', float('Inf'))
+    for pin in board.pins:
+        download_urls([pin.url], pin.id, pin.ext, float('Inf'),
+                      output_dir=output_dir, faker=True, **kwargs)
 
 
 def huaban_download(url, output_dir='.', **kwargs):

From 0dc96c29a38a8e1bc0fc494e32a2f92865146f78 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Tue, 15 Mar 2016 12:24:46 -0400
Subject: [PATCH 0081/1225] [dilidili] Delete unused code

---
 src/you_get/extractors/dilidili.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index c68c17a692..615f986104 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -33,14 +33,6 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
         stream_types.append({'id': str(i[1][-1]), 'container': 'mp4', 'video_profile': i[0]})
     return stream_types
 
-##----------------------------------------------------------------------
-#def dilidili_parser_data_to_download_url(typ ,vid ,hd2 ,sign, tmsign, ulk):
-    #"""Not used for now"""
-    #parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
-    #html = get_content(parse_url, headers=headers)
-    
-    #return match1(html, r'<file><!\[CDATA\[(.+)\]\]></file>')
-
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     if re.match(r'http://www.dilidili.com/watch/\w+', url):

From 318b74827decbef368da5727f3536ba4c8139689 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 16 Mar 2016 20:36:02 +0100
Subject: [PATCH 0082/1225] [imgur] fix for direct images

---
 src/you_get/extractor.py        |  2 +-
 src/you_get/extractors/imgur.py | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 1b547905fd..3cc78289b0 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -33,7 +33,7 @@ def __init__(self, *args):
 
     def download_by_url(self, url, **kwargs):
         self.url = url
-        self.vid= None
+        self.vid = None
 
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             set_proxy(parse_host(kwargs['extractor_proxy']))
diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 80d68ce835..20848a8705 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -37,11 +37,15 @@ def prepare(self, **kwargs):
 
         elif re.search(r'i\.imgur\.com/', self.url):
             # direct image
-            universal_download(self.url,
-                               output_dir=kwargs['output_dir'],
-                               merge=kwargs['merge'],
-                               info_only=kwargs['info_only'])
-            exit(0) # FIXME!
+            _, container, size = url_info(self.url)
+            self.streams = {
+                'original': {
+                    'src': [self.url],
+                    'size': size,
+                    'container': container
+                }
+            }
+            self.title = r1(r'i\.imgur\.com/([^./]*)', self.url)
 
         else:
             # gallery image

From b4cbd6dc6c0460da8e93523a74011357e5fa3496 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 16 Mar 2016 21:52:30 +0100
Subject: [PATCH 0083/1225] [youku] close #991

---
 src/you_get/extractors/youku.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 38ceeb0c4a..82fd6624dd 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -203,6 +203,7 @@ def prepare(self, **kwargs):
             if stream_id in stream_types and stream['audio_lang'] == audio_lang:
                 if 'alias-of' in stream_types[stream_id]:
                     stream_id = stream_types[stream_id]['alias-of']
+                if stream_id in self.streams: continue # STOP!
                 self.streams[stream_id] = {
                     'container': stream_types[stream_id]['container'],
                     'video_profile': stream_types[stream_id]['video_profile'],
@@ -217,6 +218,7 @@ def prepare(self, **kwargs):
             if stream_id in stream_types and stream['audio_lang'] == audio_lang:
                 if 'alias-of' in stream_types[stream_id]:
                     stream_id = stream_types[stream_id]['alias-of']
+                if stream_id in self.streams_fallback_parameter: continue # STOP!
                 self.streams_fallback_parameter[stream_id] = {
                     'fileid': stream['stream_fileid'],
                     'segs': stream['segs']

From 30272b737505086976b0db944c05a39c8a346d38 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 17 Mar 2016 03:00:47 +0100
Subject: [PATCH 0084/1225] [youku] fix #991 completely

---
 src/you_get/extractors/youku.py | 113 +++++++++++++-------------------
 1 file changed, 46 insertions(+), 67 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 82fd6624dd..bc6e85e35c 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -151,18 +151,12 @@ def prepare(self, **kwargs):
                 exit(0)
 
         api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
-        api_url1 = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
         try:
             meta = json.loads(get_content(
                 api_url,
                 headers={'Referer': 'http://static.youku.com/'}
             ))
-            meta1 = json.loads(get_content(
-                api_url1,
-                headers={'Referer': 'http://static.youku.com/'}
-            ))
             data = meta['data']
-            data1 = meta1['data']
             assert 'stream' in data
         except AssertionError:
             if 'error' in data:
@@ -171,17 +165,11 @@ def prepare(self, **kwargs):
                     self.password_protected = True
                     self.password = input(log.sprint('Password: ', log.YELLOW))
                     api_url += '&pwd={}'.format(self.password)
-                    api_url1 += '&pwd={}'.format(self.password)
                     meta = json.loads(get_content(
                         api_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
-                    meta1 = json.loads(get_content(
-                        api_url1,
-                        headers={'Referer': 'http://static.youku.com/'}
-                    ))
                     data = meta['data']
-                    data1 = meta1['data']
                 else:
                     log.wtf('[Failed] ' + data['error']['note'])
             else:
@@ -196,33 +184,30 @@ def prepare(self, **kwargs):
 
         stream_types = dict([(i['id'], i) for i in self.stream_types])
         self.streams_parameter = {}
-        self.streams_fallback_parameter = {}
-        audio_lang = data1['stream'][0]['audio_lang']
-        for stream in data1['stream']:
-            stream_id = stream['stream_type']
-            if stream_id in stream_types and stream['audio_lang'] == audio_lang:
-                if 'alias-of' in stream_types[stream_id]:
-                    stream_id = stream_types[stream_id]['alias-of']
-                if stream_id in self.streams: continue # STOP!
-                self.streams[stream_id] = {
-                    'container': stream_types[stream_id]['container'],
-                    'video_profile': stream_types[stream_id]['video_profile'],
-                    'size': stream['size']
-                }
-                self.streams_parameter[stream_id] = {
-                    'fileid': stream['stream_fileid'],
-                    'segs': stream['segs']
-                }
+        audio_lang = data['stream'][0]['audio_lang']
+
         for stream in data['stream']:
             stream_id = stream['stream_type']
             if stream_id in stream_types and stream['audio_lang'] == audio_lang:
                 if 'alias-of' in stream_types[stream_id]:
                     stream_id = stream_types[stream_id]['alias-of']
-                if stream_id in self.streams_fallback_parameter: continue # STOP!
-                self.streams_fallback_parameter[stream_id] = {
-                    'fileid': stream['stream_fileid'],
-                    'segs': stream['segs']
-                }
+
+                if stream_id not in self.streams:
+                    self.streams[stream_id] = {
+                        'container': stream_types[stream_id]['container'],
+                        'video_profile': stream_types[stream_id]['video_profile'],
+                        'size': stream['size'],
+                        'pieces': [{
+                            'fileid': stream['stream_fileid'],
+                            'segs': stream['segs']
+                        }]
+                    }
+                else:
+                    self.streams[stream_id]['size'] += stream['size']
+                    self.streams[stream_id]['pieces'].append({
+                        'fileid': stream['stream_fileid'],
+                        'segs': stream['segs']
+                    })
 
         # Audio languages
         if 'dvd' in data and 'audiolang' in data['dvd']:
@@ -249,42 +234,36 @@ def extract(self, **kwargs):
         )
         sid, token = e_code.split('_')
 
-        sp = self.streams_parameter
         while True:
             try:
-                segs = sp[stream_id]['segs']
-                streamfileid = sp[stream_id]['fileid']
-
                 ksegs = []
-                for no in range(0, len(segs)):
-                    k = segs[no]['key']
-                    assert k != -1
-                    fileid, ep = self.__class__.generate_ep(no, streamfileid,
-                                                            sid, token)
-                    q = parse.urlencode(dict(
-                        ctype = 12,
-                        ev    = 1,
-                        K     = k,
-                        ep    = parse.unquote(ep),
-                        oip   = str(self.ip),
-                        token = token,
-                        yxon  = 1
-                    ))
-                    u = 'http://k.youku.com/player/getFlvPath/sid/{sid}_00' \
-                        '/st/{container}/fileid/{fileid}?{q}'.format(
-                        sid       = sid,
-                        container = self.streams[stream_id]['container'],
-                        fileid    = fileid,
-                        q         = q
-                    )
-                    # unset_proxy()  also strips cookies,because k.youku.com doesn't need cookies('r') for now
-                    ksegs += [i['server'] for i in json.loads(get_content(u))]
-            except error.HTTPError as e:
-                # Use fallback stream data in case of HTTP 404
-                log.e('[Error] ' + str(e))
-                sp = self.streams_fallback_parameter
-            except KeyError:
-                # Move on to next stream if best quality not available
+                pieces = self.streams[stream_id]['pieces']
+                for piece in pieces:
+                    segs = piece['segs']
+                    streamfileid = piece['fileid']
+                    for no in range(0, len(segs)):
+                        k = segs[no]['key']
+                        fileid, ep = self.__class__.generate_ep(no, streamfileid,
+                                                                sid, token)
+                        q = parse.urlencode(dict(
+                            ctype = 12,
+                            ev    = 1,
+                            K     = k,
+                            ep    = parse.unquote(ep),
+                            oip   = str(self.ip),
+                            token = token,
+                            yxon  = 1
+                        ))
+                        u = 'http://k.youku.com/player/getFlvPath/sid/{sid}_00' \
+                            '/st/{container}/fileid/{fileid}?{q}'.format(
+                                sid       = sid,
+                                container = self.streams[stream_id]['container'],
+                                fileid    = fileid,
+                                q         = q
+                            )
+                        ksegs += [i['server'] for i in json.loads(get_content(u))]
+            except:
+                # Move on to next stream
                 del self.streams_sorted[0]
                 stream_id = self.streams_sorted[0]['id']
             else: break

From c9223acd11bff0d19899eaa8fb0f1dddadc664b0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 17 Mar 2016 03:24:16 +0100
Subject: [PATCH 0085/1225] version 0.4.350

---
 src/you_get/extractors/youku.py | 1 -
 src/you_get/version.py          | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index bc6e85e35c..a5c369b22c 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -183,7 +183,6 @@ def prepare(self, **kwargs):
             log.wtf('[Failed] Wrong password.')
 
         stream_types = dict([(i['id'], i) for i in self.stream_types])
-        self.streams_parameter = {}
         audio_lang = data['stream'][0]['audio_lang']
 
         for stream in data['stream']:
diff --git a/src/you_get/version.py b/src/you_get/version.py
index 99aeb880e0..a9872b96da 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.330'
+__version__ = '0.4.350'

From 9f9944301cadcbd87cbb744ab4ca601eb68b05b7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Mar 2016 01:20:58 +0100
Subject: [PATCH 0086/1225] [755] do not fail the whole process

---
 src/you_get/extractors/nanagogo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index 6382afc444..222659f6fb 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -35,6 +35,7 @@ def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                           'size':  size})
 
     size = sum([i['size'] for i in items])
+    if size == 0: return # do not fail the whole process
     print_info(site_info, title, ext, size)
     if not info_only:
         for i in items:

From 68b340e19b2978cc04b089a427b52ca1b1a93990 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Mar 2016 03:13:01 +0100
Subject: [PATCH 0087/1225] [imgur] fix for album links

Example:
- http://imgur.com/a/6CrHX
---
 src/you_get/extractors/imgur.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 20848a8705..b7bdc4c5bd 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -16,7 +16,9 @@ def prepare(self, **kwargs):
         if re.search(r'imgur\.com/a/', self.url):
             # album
             content = get_content(self.url)
-            album = json.loads(match1(content, r'album\s*:\s*({.*}),'))
+            album = match1(content, r'album\s*:\s*({.*}),') or \
+                    match1(content, r'image\s*:\s*({.*}),')
+            album = json.loads(album)
             count = album['album_images']['count']
             images = album['album_images']['images']
             ext = images[0]['ext']

From c054da9535a0b010b473d4427a0c414efe157651 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Mar 2016 17:08:18 +0100
Subject: [PATCH 0088/1225] [youku] fix #991 (again)

---
 src/you_get/extractors/youku.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index a5c369b22c..3d785e1ebb 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -150,13 +150,19 @@ def prepare(self, **kwargs):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
-        api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
+        api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
+        api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
         try:
             meta = json.loads(get_content(
                 api_url,
                 headers={'Referer': 'http://static.youku.com/'}
             ))
+            meta12 = json.loads(get_content(
+                api12_url,
+                headers={'Referer': 'http://static.youku.com/'}
+            ))
             data = meta['data']
+            data12 = meta12['data']
             assert 'stream' in data
         except AssertionError:
             if 'error' in data:
@@ -165,19 +171,25 @@ def prepare(self, **kwargs):
                     self.password_protected = True
                     self.password = input(log.sprint('Password: ', log.YELLOW))
                     api_url += '&pwd={}'.format(self.password)
+                    api_url12 += '&pwd={}'.format(self.password)
                     meta = json.loads(get_content(
                         api_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
+                    meta12 = json.loads(get_content(
+                        api_url12,
+                        headers={'Referer': 'http://static.youku.com/'}
+                    ))
                     data = meta['data']
+                    data12 = meta12['data']
                 else:
                     log.wtf('[Failed] ' + data['error']['note'])
             else:
                 log.wtf('[Failed] Video not found.')
 
         self.title = data['video']['title']
-        self.ep = data['security']['encrypt_string']
-        self.ip = data['security']['ip']
+        self.ep = data12['security']['encrypt_string']
+        self.ip = data12['security']['ip']
 
         if 'stream' not in data and self.password_protected:
             log.wtf('[Failed] Wrong password.')

From 3be2f06877308a6ffb8d2592b6b4fa0881ea4f71 Mon Sep 17 00:00:00 2001
From: lh <5linhua5@gmail.com>
Date: Fri, 18 Mar 2016 23:58:51 +0800
Subject: [PATCH 0089/1225] [Youku] Recover youku for stream_fallback

---
 src/you_get/extractors/youku.py | 34 +++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 3d785e1ebb..cefb771971 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -220,6 +220,30 @@ def prepare(self, **kwargs):
                         'segs': stream['segs']
                     })
 
+        self.streams_fallback = {}
+        for stream in data12['stream']:
+            stream_id = stream['stream_type']
+            if stream_id in stream_types and stream['audio_lang'] == audio_lang:
+                if 'alias-of' in stream_types[stream_id]:
+                    stream_id = stream_types[stream_id]['alias-of']
+
+                if stream_id not in self.streams_fallback:
+                    self.streams_fallback[stream_id] = {
+                        'container': stream_types[stream_id]['container'],
+                        'video_profile': stream_types[stream_id]['video_profile'],
+                        'size': stream['size'],
+                        'pieces': [{
+                            'fileid': stream['stream_fileid'],
+                            'segs': stream['segs']
+                        }]
+                    }
+                else:
+                    self.streams_fallback[stream_id]['size'] += stream['size']
+                    self.streams_fallback[stream_id]['pieces'].append({
+                        'fileid': stream['stream_fileid'],
+                        'segs': stream['segs']
+                    })
+
         # Audio languages
         if 'dvd' in data and 'audiolang' in data['dvd']:
             self.audiolang = data['dvd']['audiolang']
@@ -254,6 +278,7 @@ def extract(self, **kwargs):
                     streamfileid = piece['fileid']
                     for no in range(0, len(segs)):
                         k = segs[no]['key']
+                        assert k != -1
                         fileid, ep = self.__class__.generate_ep(no, streamfileid,
                                                                 sid, token)
                         q = parse.urlencode(dict(
@@ -273,8 +298,13 @@ def extract(self, **kwargs):
                                 q         = q
                             )
                         ksegs += [i['server'] for i in json.loads(get_content(u))]
-            except:
-                # Move on to next stream
+            except error.HTTPError as e:
+                # Use fallback stream data in case of HTTP 404
+                log.e('[Error] ' + str(e))
+                self.streams = {}
+                self.streams = self.streams_fallback
+            except KeyError:
+                # Move on to next stream if best quality not available
                 del self.streams_sorted[0]
                 stream_id = self.streams_sorted[0]['id']
             else: break

From 40968e8e89b40adbff31b1aaeadea60d9efadcf2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 19 Mar 2016 04:19:07 +0100
Subject: [PATCH 0090/1225] [twitter] fix for GIF videos Example: -
 https://twitter.com/potatochef_/status/709229020441743360

---
 src/you_get/extractors/twitter.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 91456491f3..251cb8d53b 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -44,6 +44,8 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         # always use i/cards or videos url
         if not re.match(r'https?://twitter.com/i/', url):
             url = r1(r'<meta\s*property="og:video:url"\s*content="([^"]+)"', html)
+            if not url:
+                url = 'https://twitter.com/i/videos/%s' % item_id
             html = get_content(url)
 
         data_config = r1(r'data-config="([^"]*)"', html) or \

From ebdc27bce5f3fcea8da37248e24b5c14d9e10fe7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 19 Mar 2016 05:28:26 +0100
Subject: [PATCH 0091/1225] [zhanqi] revert to eaab0ce, close #992

---
 src/you_get/extractors/zhanqi.py | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index f450f9549e..7d6b75b67c 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -4,6 +4,10 @@
 
 from ..common import *
 import re
+import base64
+import json
+import time
+import hashlib
 
 def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_content(url)
@@ -19,15 +23,33 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
     title = unescape_html(title)
     rtmp_base = "http://wshdl.load.cdn.zhanqi.tv/zqlive"
     vod_base = "http://dlvod.cdn.zhanqi.tv"
+    rtmp_real_base = "rtmp://dlrtmp.cdn.zhanqi.tv/zqlive/"
+    room_info = "http://www.zhanqi.tv/api/static/live.roomid/"
+    KEY_MASK = "#{&..?!("
+    ak2_pattern = r'ak2":"\d-([^|]+)'
     
     if video_type == "LIVE":
         rtmp_id = match1(html, rtmp_id_patt).replace('\\/','/')
-        request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
-        real_url = get_html(request_url)
+        #request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
+        #real_url = get_html(request_url)
+        html2 = get_content(room_info + rtmp_id.split("_")[0] + ".json")
+        json_data = json.loads(html2)
+        cdns = json_data["data"]["flashvars"]["cdns"]
+        cdns = base64.b64decode(cdns).decode("utf-8")
+        cdn = match1(cdns, ak2_pattern)
+        cdn = base64.b64decode(cdn).decode("utf-8")
+        key = ''
+        i = 0
+        while(i < len(cdn)):
+            key = key + chr(ord(cdn[i]) ^ ord(KEY_MASK[i % 8]))
+            i = i + 1
+        time_hex = hex(int(time.time()))[2:]
+        key = hashlib.md5(bytes(key + "/zqlive/" + rtmp_id + time_hex, "utf-8")).hexdigest()
+        real_url = rtmp_real_base + '/' + rtmp_id + "?k=" + key + "&t=" + time_hex
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
-            #download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
-            download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+            download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
+            #download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
     elif video_type == "VOD":
         vod_m3u8_request = vod_base + match1(html, vod_m3u8_id_patt).replace('\\/','/')
         vod_m3u8 = get_html(vod_m3u8_request)
@@ -47,6 +69,7 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
             download_urls(real_url, title, type_ or 'ts', size, output_dir, merge = merge)
     else:
         NotImplementedError('Unknown_video_type')
+
 site_info = "zhanqi.tv"
 download = zhanqi_download
 download_playlist = playlist_not_supported('zhanqi')

From d02b5bb9ad075e96236c37a7abd245b2532681f0 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Sat, 19 Mar 2016 22:51:09 -0700
Subject: [PATCH 0092/1225] common.py: Print URL of get_content call when
 --debug is enabled

Use logging, and raise the root logger level from WARNING (default) to
DEBUG when --debug is enabled.

This logging facility can be extended to other functions.
---
 src/you_get/common.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f15481a388..a417bcc713 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -91,6 +91,7 @@
 import getopt
 import json
 import locale
+import logging
 import os
 import platform
 import re
@@ -298,6 +299,8 @@ def get_content(url, headers={}, decoded=True):
         The content as a string.
     """
 
+    logging.debug('get_content: %s' % url)
+
     req = request.Request(url, headers=headers)
     if cookies:
         cookies.add_cookie_header(req)
@@ -1032,6 +1035,8 @@ def version():
               % get_version(kwargs['repo_path']
             if 'repo_path' in kwargs else __version__))
 
+    logging.basicConfig(format='[%(levelname)s] %(message)s')
+
     help = 'Usage: %s [OPTION]... [URL]...\n\n' % script_name
     help += '''Startup options:
     -V | --version                      Print version and exit.
@@ -1055,7 +1060,7 @@ def version():
     -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
          --no-proxy                     Never use a proxy.
-    -d | --debug                        Show traceback for debugging.
+    -d | --debug                        Show traceback and other debug info.
     '''
 
     short_opts = 'Vhfiuc:ndF:O:o:p:x:y:'
@@ -1145,6 +1150,8 @@ def version():
             proxy = ''
         elif o in ('-d', '--debug'):
             traceback = True
+            # Set level of root logger to DEBUG
+            logging.getLogger().setLevel(logging.DEBUG)
         elif o in ('-F', '--format', '--stream', '--itag'):
             stream_id = a
         elif o in ('-O', '--output-filename'):

From c4491551b2d22999bc0d2b1cd3598c945c1a2c86 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 21 Mar 2016 16:38:36 +0100
Subject: [PATCH 0093/1225] [youku] (partly) fix #996

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index cefb771971..b4fc38a7dd 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -278,7 +278,7 @@ def extract(self, **kwargs):
                     streamfileid = piece['fileid']
                     for no in range(0, len(segs)):
                         k = segs[no]['key']
-                        assert k != -1
+                        if k == -1: break # we hit the paywall; stop here
                         fileid, ep = self.__class__.generate_ep(no, streamfileid,
                                                                 sid, token)
                         q = parse.urlencode(dict(

From 0cabb980b65316a9b969b675f99d0b89dea25175 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 22 Mar 2016 12:46:04 +0100
Subject: [PATCH 0094/1225] [iqiyi] update key (close #1005)

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index e3cca88dad..7431b91fdc 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -45,7 +45,7 @@
 
 '''
 def mix(tvid):
-    salt = '8ed797d224d043e7ac23d95b70227d32'
+    salt = '4a1caba4b4465345366f28da7c117d20'
     tm = str(randint(2000,4000))
     sc = hashlib.new('md5', bytes(salt + tm + tvid, 'utf-8')).hexdigest()
     return tm, sc, 'eknas'

From 39ceacd8d6cbbafc47b31408a8817a1dc35f7998 Mon Sep 17 00:00:00 2001
From: Botu Sun <sunbotu@gmail.com>
Date: Fri, 25 Mar 2016 16:44:32 +0900
Subject: [PATCH 0095/1225] fix to handle new domain of douyu.com (was
 douyutv.com)

---
 src/you_get/common.py             | 2 +-
 src/you_get/extractors/douyutv.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a417bcc713..ccfd8c27e5 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -15,7 +15,7 @@
     'dilidili'         : 'dilidili',
     'dongting'         : 'dongting',
     'douban'           : 'douban',
-    'douyutv'          : 'douyutv',
+    'douyu'            : 'douyutv',
     'ehow'             : 'ehow',
     'facebook'         : 'facebook',
     'fc2'              : 'fc2video',
diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 023a7249c7..449022caa6 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -12,7 +12,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     #Thanks to @yan12125 for providing decoding method!!
     suffix = 'room/%s?aid=android&client_sys=android&time=%d' % (room_id, int(time.time()))
     sign = hashlib.md5((suffix + '1231').encode('ascii')).hexdigest()
-    json_request_url = "http://www.douyutv.com/api/v1/%s&auth=%s" % (suffix, sign)
+    json_request_url = "http://www.douyu.com/api/v1/%s&auth=%s" % (suffix, sign)
     content = get_html(json_request_url)
     data = json.loads(content)['data']
     server_status = data.get('error',0)
@@ -28,6 +28,6 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     if not info_only:
         download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
 
-site_info = "douyutv.com"
+site_info = "douyu.com"
 download = douyutv_download
-download_playlist = playlist_not_supported('douyutv')
+download_playlist = playlist_not_supported('douyu')

From bca2c3eed387125296f14d7544ba9887065bf1d8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Mar 2016 16:49:03 +0100
Subject: [PATCH 0096/1225] [util.strings] use 4-space indentation

---
 src/you_get/util/strings.py | 34 +++++++++++++++++-----------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/src/you_get/util/strings.py b/src/you_get/util/strings.py
index 7e74f35e2a..a81cc15c82 100644
--- a/src/you_get/util/strings.py
+++ b/src/you_get/util/strings.py
@@ -1,25 +1,25 @@
 try:
-  # py 3.4
-  from html import unescape as unescape_html
+    # py 3.4
+    from html import unescape as unescape_html
 except ImportError:
-  import re
-  from html.entities import entitydefs
+    import re
+    from html.entities import entitydefs
 
-  def unescape_html(string):
-    '''HTML entity decode'''
-    string = re.sub(r'&#[^;]+;', _sharp2uni, string)
-    string = re.sub(r'&[^;]+;', lambda m: entitydefs[m.group(0)[1:-1]], string)
-    return string
+    def unescape_html(string):
+        '''HTML entity decode'''
+        string = re.sub(r'&#[^;]+;', _sharp2uni, string)
+        string = re.sub(r'&[^;]+;', lambda m: entitydefs[m.group(0)[1:-1]], string)
+        return string
 
-  def _sharp2uni(m):
-    '''&#...; ==> unicode'''
-    s = m.group(0)[2:].rstrip(';；')
-    if s.startswith('x'):
-      return chr(int('0'+s, 16))
-    else:
-      return chr(int(s))
+    def _sharp2uni(m):
+        '''&#...; ==> unicode'''
+        s = m.group(0)[2:].rstrip(';；')
+        if s.startswith('x'):
+            return chr(int('0'+s, 16))
+        else:
+            return chr(int(s))
 
 from .fs import legitimize
 
 def get_filename(htmlstring):
-  return legitimize(unescape_html(htmlstring))
+    return legitimize(unescape_html(htmlstring))

From 7af203e8e346ae3b8bedd45e92dc4aa25b19cb7d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Mar 2016 16:49:28 +0100
Subject: [PATCH 0097/1225] [util.strings] add parameterize()

---
 src/you_get/util/strings.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/util/strings.py b/src/you_get/util/strings.py
index a81cc15c82..26d555942c 100644
--- a/src/you_get/util/strings.py
+++ b/src/you_get/util/strings.py
@@ -23,3 +23,6 @@ def _sharp2uni(m):
 
 def get_filename(htmlstring):
     return legitimize(unescape_html(htmlstring))
+
+def parameterize(string):
+    return "'%s'" % string.replace("'", r"'\''")

From 46a1f51a8f8c0bd308d69ea32e4e63082c656a9f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Mar 2016 16:56:08 +0100
Subject: [PATCH 0098/1225] [ffmpeg] fix #1009

---
 src/you_get/processor/ffmpeg.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 7262c46218..e4ea1223ba 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -2,6 +2,7 @@
 
 import os.path
 import subprocess
+from ..util.strings import parameterize
 
 def get_usable_ffmpeg(cmd):
     try:
@@ -53,7 +54,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         concat_list = open(output + '.txt', 'w', encoding="utf-8")
         for file in files:
             if os.path.isfile(file):
-                concat_list.write("file '%s'\n" % file)
+                concat_list.write("file %s\n" % parameterize(file))
         concat_list.close()
 
         params = [FFMPEG] + LOGLEVEL
@@ -118,7 +119,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
             if os.path.isfile(file):
                 # for escaping rules, see:
                 # https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping
-                concat_list.write("file '%s'\n" % file.replace("'", r"'\''"))
+                concat_list.write("file %s\n" % parameterize(file))
         concat_list.close()
 
         params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-y', '-i']
@@ -163,7 +164,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         concat_list = open(output + '.txt', 'w', encoding="utf-8")
         for file in files:
             if os.path.isfile(file):
-                concat_list.write("file '%s'\n" % file)
+                concat_list.write("file %s\n" % parameterize(file))
         concat_list.close()
 
         params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-y', '-i']

From 4c012aeba4fec035f14696b81eb65fce1fd703bb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Mar 2016 16:56:54 +0100
Subject: [PATCH 0099/1225] [common] do not download captions with --player

---
 src/you_get/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a417bcc713..9c732693ac 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1160,6 +1160,7 @@ def version():
             output_dir = a
         elif o in ('-p', '--player'):
             player = a
+            caption = False
         elif o in ('-x', '--http-proxy'):
             proxy = a
         elif o in ('-y', '--extractor-proxy'):

From ea0b81ad1e56935e14429e3b064300b679c61ce1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Mar 2016 17:24:40 +0100
Subject: [PATCH 0100/1225] version 0.4.365

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index a9872b96da..d8db720e87 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.350'
+__version__ = '0.4.365'

From 136f16445258d62ed62a0e013ecd2ed6c7969467 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 30 Mar 2016 21:37:21 +0200
Subject: [PATCH 0101/1225] [bilibili] download multi parts by default

---
 src/you_get/extractors/bilibili.py | 22 +++++++++-------------
 1 file changed, 9 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index cd9186029d..1a6ca325dc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -127,7 +127,8 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         if re.match(r'https?://live\.bilibili\.com/', url):
             title = r1(r'<title>([^<>]+)</title>', html)
             bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-        elif 'playlist' in kwargs and kwargs['playlist']:
+
+        else:
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
@@ -140,15 +141,18 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                 if flashvars:
                     t, cid = flashvars.split('=', 1)
                     cids.append(cid.split('&')[0])
+
+            # no multi-P
+            if not pages:
+                cids = [cid]
+                titles = [r1(r'<option value=.* selected>(.+)</option>', html) or title]
+
             for i in range(len(cids)):
                 bilibili_download_by_cid(cids[i],
                                          titles[i],
                                          output_dir=output_dir,
                                          merge=merge,
                                          info_only=info_only)
-        else:
-            title = r1(r'<option value=.* selected>(.+)</option>', html) or title
-            bilibili_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
 
     elif t == 'vid':
         sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
@@ -169,14 +173,6 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
-def bilibili_download_playlist(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    bilibili_download(url,
-                      output_dir=output_dir,
-                      merge=merge,
-                      info_only=info_only,
-                      playlist=True,
-                      **kwargs)
-
 site_info = "bilibili.com"
 download = bilibili_download
-download_playlist = bilibili_download_playlist
+download_playlist = bilibili_download

From 6b9e2978908fa1f55866b4c65d619756adc08cfe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 31 Mar 2016 17:42:00 +0200
Subject: [PATCH 0102/1225] [embed] support netease, close #1001

---
 src/you_get/extractors/embed.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index b594b9f48c..fd463c92bc 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -4,6 +4,7 @@
 
 from .iqiyi import iqiyi_download_by_vid
 from .le import letvcloud_download_by_vu
+from .netease import netease_download
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
@@ -36,10 +37,13 @@
 
 iqiyi_embed_patterns = [ 'player\.video\.qiyi\.com/([^/]+)/[^/]+/[^/]+/[^/]+\.swf[^"]+tvId=(\d+)' ]
 
+netease_embed_patterns = [ '(http://\w+\.163\.com/movie/[^\'"]+)' ]
+
 def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
-    content = get_content(url)
+    content = get_content(url, headers=fake_headers)
     found = False
     title = match1(content, '<title>([^<>]+)</title>')
+
     vids = matchall(content, youku_embed_patterns)
     for vid in set(vids):
         found = True
@@ -60,6 +64,11 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         iqiyi_download_by_vid((vid[1], vid[0]), title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
+    urls = matchall(content, netease_embed_patterns)
+    for url in urls:
+        found = True
+        netease_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+
     if not found:
         raise NotImplementedError(url)
 

From 53cb97def5f2aa760de32f652e8db7be76be529a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 5 Apr 2016 00:39:28 +0200
Subject: [PATCH 0103/1225] [processor.ffmpeg] set safe option to -1 (as
 default value changed from -1 to 1 in ffmpeg 3.0.1), fix #1023

---
 src/you_get/processor/ffmpeg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index e4ea1223ba..320eb6421c 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -58,7 +58,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         concat_list.close()
 
         params = [FFMPEG] + LOGLEVEL
-        params.extend(['-f', 'concat', '-y', '-i'])
+        params.extend(['-f', 'concat', '-safe', '-1', '-y', '-i'])
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
@@ -122,7 +122,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
                 concat_list.write("file %s\n" % parameterize(file))
         concat_list.close()
 
-        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-y', '-i']
+        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 
@@ -167,7 +167,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
                 concat_list.write("file %s\n" % parameterize(file))
         concat_list.close()
 
-        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-y', '-i']
+        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
         params.append(output + '.txt')
         params += ['-c', 'copy', output]
 

From 878d1d56c40ddb9783430f0677afd52db46740be Mon Sep 17 00:00:00 2001
From: Timothy Cyrus <tcyrus@users.noreply.github.com>
Date: Fri, 8 Apr 2016 10:31:27 -0400
Subject: [PATCH 0104/1225] Update README.md

Changed Badges from PNG to SVG
Also Fixed Badge Links
---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index abdf39e531..ddfac26d35 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # You-Get
 
-[![PyPI version](https://badge.fury.io/py/you-get.png)](http://badge.fury.io/py/you-get)
-[![Build Status](https://api.travis-ci.org/soimort/you-get.png)](https://travis-ci.org/soimort/you-get)
+[![PyPI version](https://img.shields.io/pypi/v/you-get.svg)](https://pypi.python.org/pypi/you-get/)
+[![Build Status](https://travis-ci.org/soimort/you-get.svg)](https://travis-ci.org/soimort/you-get)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 [You-Get](https://you-get.org/) is a tiny command-line utility to download media contents (videos, audios, images) from the Web, in case there is no other handy way to do it.

From cda4d2ae8906789a307a54d4b72c616af96ea9b6 Mon Sep 17 00:00:00 2001
From: Rokic <Rokic.github@gmail.com>
Date: Mon, 11 Apr 2016 22:55:18 +0800
Subject: [PATCH 0105/1225] [Pornhub] Add Support, Universal Solution

Bypass anti-spider strategies of Pornhub.
---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a8bda396ee..e20be32b11 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1254,7 +1254,7 @@ def url_to_module(url):
     else:
         import http.client
         conn = http.client.HTTPConnection(video_host)
-        conn.request("HEAD", video_url)
+        conn.request("HEAD", video_url, headers=fake_headers)
         res = conn.getresponse()
         location = res.getheader('location')
         if location and location != url and not location.startswith('/'):

From 244f77e6b72ddf68afcdcafdec56efeca87521a5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 13 Apr 2016 17:33:34 +0200
Subject: [PATCH 0106/1225] Antigen: call python3 explicitly

---
 you-get.plugin.zsh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/you-get.plugin.zsh b/you-get.plugin.zsh
index 70f2f48495..6282eda57a 100644
--- a/you-get.plugin.zsh
+++ b/you-get.plugin.zsh
@@ -1,3 +1,3 @@
 #!/usr/bin/env zsh
-alias you-get="noglob $(dirname $0)/you-get"
-alias you-vlc="noglob $(dirname $0)/you-get --player vlc"
+alias you-get="noglob python3 $(dirname $0)/you-get"
+alias you-vlc="noglob python3 $(dirname $0)/you-get --player vlc"

From 0bd78f5b186957b16ace8659d07da6f873633606 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 15 Apr 2016 11:25:27 +0800
Subject: [PATCH 0107/1225] fix v.qq.com 403 code

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index a0d80664c5..d6684a3e65 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,7 +5,7 @@
 from ..common import *
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    api = "http://h5vv.video.qq.com/getinfo?otype=json&vid=%s" % vid
+    api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
     output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
     url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']

From 0424cd9f8c54eb5eb036cdff4b07a7bdc64c6b21 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Tue, 19 Apr 2016 17:38:27 +0800
Subject: [PATCH 0108/1225] support for redirect URLs

---
 src/you_get/extractors/qq.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index d6684a3e65..47ca663e17 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -18,7 +18,13 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    if 'iframe/player.html' in url:
+    if 'v.qq.com/page' in url:
+        # for URLs like this:
+        # http://v.qq.com/page/k/9/7/k0194pwgw97.html
+        # it will redirect.
+        vid = match1(url, r'\b(\w+).html')
+        title = vid
+    elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is
         title = vid

From 17393a5fc7bbe36ee16085a86fa653b79d46e4db Mon Sep 17 00:00:00 2001
From: yuderbin <591558148@qq.com>
Date: Tue, 19 Apr 2016 18:33:38 +0800
Subject: [PATCH 0109/1225] add support to kuaibao.qq.com

---
 src/you_get/extractors/qq.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 47ca663e17..00a99cd96f 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -24,6 +24,10 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         # it will redirect.
         vid = match1(url, r'\b(\w+).html')
         title = vid
+    elif 'kuaibao.qq.com' in url:
+        content = get_html(url)
+        vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
+        title = match1(content, r'title">([^"]+)</p>').strip()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From cc49724b76674c51ebd8b43d3bd2e9c254062f2a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 24 Apr 2016 01:43:37 +0200
Subject: [PATCH 0110/1225] [youku] fix typos in c054da95

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index b4fc38a7dd..3135e82251 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -171,13 +171,13 @@ def prepare(self, **kwargs):
                     self.password_protected = True
                     self.password = input(log.sprint('Password: ', log.YELLOW))
                     api_url += '&pwd={}'.format(self.password)
-                    api_url12 += '&pwd={}'.format(self.password)
+                    api12_url += '&pwd={}'.format(self.password)
                     meta = json.loads(get_content(
                         api_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
                     meta12 = json.loads(get_content(
-                        api_url12,
+                        api12_url,
                         headers={'Referer': 'http://static.youku.com/'}
                     ))
                     data = meta['data']

From b919fdd8013088daa4f28bdd8e7ed1a4ff8577d5 Mon Sep 17 00:00:00 2001
From: Danieliu <liudanking@gmail.com>
Date: Wed, 27 Apr 2016 17:13:12 +0800
Subject: [PATCH 0111/1225] Update you-get

fix python version
---
 you-get | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/you-get b/you-get
index fbbdb9f266..85f3f754b7 100755
--- a/you-get
+++ b/you-get
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os, sys
 
 _srcdir = 'src/'

From 1a018c5261961f86dacb5c0f17648b8daad5b987 Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Wed, 27 Apr 2016 17:08:46 -0400
Subject: [PATCH 0112/1225] [Dilidili] Fix codec, fix #1085

---
 src/you_get/extractors/dilidili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 615f986104..6bc6ae2831 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -40,11 +40,11 @@ def dilidili_download(url, output_dir = '.', merge = False, info_only = False, *
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         
         # player loaded via internal iframe
-        frame_url = re.search(r'<iframe (.+)src="(.+)\" f(.+)</iframe>', html).group(2)
+        frame_url = re.search(r'<iframe src=\"(.+?)\"', html).group(1)
         #print(frame_url)
         
         #https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
-        html = get_content(frame_url, headers=headers)
+        html = get_content(frame_url, headers=headers, decoded=False).decode('utf-8')
         
         match = re.search(r'(.+?)var video =(.+?);', html)
         vid = match1(html, r'var vid="(.+)"')

From 58efc40ccb95149027a24098bba8e2db95bf0d9a Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Thu, 28 Apr 2016 01:27:44 -0400
Subject: [PATCH 0113/1225] [Youku]Add COOP platform

---
 src/you_get/extractors/youku.py | 63 ++++++++++++++++++++++++++++-----
 1 file changed, 54 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 3135e82251..fefaf5ee68 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -28,7 +28,11 @@ class Youku(VideoExtractor):
     f_code_1 = 'becaf9be'
     f_code_2 = 'bf7e5f01'
 
+    ctype = 12  #differ from 86
+
     def trans_e(a, c):
+        """str, str->str
+        This is an RC4 encryption."""
         f = h = 0
         b = list(range(256))
         result = ''
@@ -49,14 +53,14 @@ def trans_e(a, c):
 
         return result
 
-    def generate_ep(no, streamfileids, sid, token):
+    def generate_ep(self, no, streamfileids, sid, token):
         number = hex(int(str(no), 10))[2:].upper()
         if len(number) == 1:
             number = '0' + number
         fileid = streamfileids[0:8] + number + streamfileids[10:]
         ep = parse.quote(base64.b64encode(
-            ''.join(Youku.trans_e(
-                Youku.f_code_2,
+            ''.join(self.__class__.trans_e(
+                self.f_code_2,  #use the 86 fcode if using 86
                 sid + '_' + fileid + '_' + token)).encode('latin1')),
             safe='~()*!.\''
         )
@@ -150,8 +154,17 @@ def prepare(self, **kwargs):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
-        api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
-        api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
+        #HACK!
+        if 'api_url' in kwargs:
+            api_url = kwargs['api_url']  #85
+            api12_url = kwargs['api12_url']  #86
+            self.ctype = kwargs['ctype']
+            self.title = kwargs['title']
+            
+        else:
+            api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
+            api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
+
         try:
             meta = json.loads(get_content(
                 api_url,
@@ -187,7 +200,8 @@ def prepare(self, **kwargs):
             else:
                 log.wtf('[Failed] Video not found.')
 
-        self.title = data['video']['title']
+        if not self.title:  #86
+            self.title = data['video']['title']
         self.ep = data12['security']['encrypt_string']
         self.ip = data12['security']['ip']
 
@@ -264,7 +278,7 @@ def extract(self, **kwargs):
             stream_id = self.streams_sorted[0]['id']
 
         e_code = self.__class__.trans_e(
-            self.__class__.f_code_1,
+            self.f_code_1,
             base64.b64decode(bytes(self.ep, 'ascii'))
         )
         sid, token = e_code.split('_')
@@ -279,10 +293,10 @@ def extract(self, **kwargs):
                     for no in range(0, len(segs)):
                         k = segs[no]['key']
                         if k == -1: break # we hit the paywall; stop here
-                        fileid, ep = self.__class__.generate_ep(no, streamfileid,
+                        fileid, ep = self.__class__.generate_ep(self, no, streamfileid,
                                                                 sid, token)
                         q = parse.urlencode(dict(
-                            ctype = 12,
+                            ctype = self.ctype,
                             ev    = 1,
                             K     = k,
                             ep    = parse.unquote(ep),
@@ -312,9 +326,40 @@ def extract(self, **kwargs):
         if not kwargs['info_only']:
             self.streams[stream_id]['src'] = ksegs
 
+    def open_download_by_vid(self, client_id, vid, **kwargs):
+        """self, str, str, **kwargs->None
+        Override the original one with VideoExtractor.
+        Most of the credit are to @ERioK, who gave his POC."""
+        self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
+        self.f_code_2 = 'msjv7h2b'
+        self.url = None
+        self.vid = vid
+        self.name = "优酷开放平台 (Youku COOP)"
+
+        #A little bit of work before self.prepare
+        sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+        playsign = json.loads(get_content(sign_url))['playsign']
+    
+        api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
+        api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
+        
+        self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
+        if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
+            unset_proxy()
+
+        try:
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+        except:
+            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
+
+        self.extract(**kwargs)
+
+        self.download(**kwargs)
+
 site = Youku()
 download = site.download_by_url
 download_playlist = site.download_playlist_by_url
 
 youku_download_by_vid = site.download_by_vid
+youku_open_download_by_vid = site.open_download_by_vid
 # Used by: acfun.py bilibili.py miomio.py tudou.py

From 3cc6e7bc661ef4e126a74d6948f154d7b500f4ba Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Thu, 28 Apr 2016 01:29:22 -0400
Subject: [PATCH 0114/1225] [Acfun]Fix Acfun Youku COOP, fix #1069

---
 src/you_get/extractors/acfun.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/acfun.py

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
old mode 100644
new mode 100755
index e78d363627..983606d42f
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -8,7 +8,7 @@
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_iid
-from .youku import youku_download_by_vid
+from .youku import youku_download_by_vid, youku_open_download_by_vid
 
 import json, re
 
@@ -32,14 +32,11 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     elif sourceType == 'letv':
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
-        a = 'http://api.aixifan.com/plays/%s/realSource' % vid
-        s = json.loads(get_content(a, headers={'deviceType': '1'}))
-        urls = s['data']['files'][-1]['url']
-        size = urls_size(urls)
-        print_info(site_info, title, 'mp4', size)
-        if not info_only:
-            download_urls(urls, title, 'mp4', size,
-                          output_dir=output_dir, merge=merge)
+        a = 'http://api.aixifan.com/plays/%s' % vid
+        s = json.loads(get_content(a, headers={'deviceType': '2'}))
+        if s['data']['source'] == "zhuzhan-youku":
+            sourceId = s['data']['sourceId']
+            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
     else:
         raise NotImplementedError(sourceType)
 

From f9c20792fc0ca2b29f0d9b18701abe5ab65b57ef Mon Sep 17 00:00:00 2001
From: David Zhuang <i+david.zhuang-mail.utoronto.ca@cnbeining.com>
Date: Thu, 28 Apr 2016 02:10:11 -0400
Subject: [PATCH 0115/1225] [Tudou]Fix acfun.tudou.com redirect, fix #695

---
 src/you_get/extractors/tudou.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index d6a7f815c8..6bbbc12bb5 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -4,6 +4,7 @@
 
 from ..common import *
 from xml.dom.minidom import parseString
+import you_get.extractors.acfun
 
 def tudou_download_by_iid(iid, title, output_dir = '.', merge = True, info_only = False):
     data = json.loads(get_decoded_html('http://www.tudou.com/outplay/goto/getItemSegs.action?iid=%s' % iid))
@@ -29,6 +30,13 @@ def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only =
     tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    if 'acfun.tudou.com' in url:  #wrong way!
+        url = url.replace('acfun.tudou.com', 'www.acfun.tv')
+        you_get.extractors.acfun.acfun_download(url, output_dir, 
+                                               merge, 
+                                               info_only)
+        return  #throw you back
+    
     # Embedded player
     id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
     if id:

From 19bd7a4a48074acfc389c0e653677e64e5e3d8b9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 29 Apr 2016 12:27:05 +0200
Subject: [PATCH 0116/1225] [youtube] fix #1092

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 2cfd1ec8ae..0ef390ed2d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -56,7 +56,7 @@ def tr_js(code):
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
-        f1def = 'function %s%s' % (re.escape(f1), f1def)
+        f1def = 'function %s%s' % (f1, f1def)
         code = tr_js(f1def)
         f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
         for f2 in f2s:

From b19bc2d5db73166f9efc0181ba1b2ca737e6ac22 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 30 Apr 2016 03:13:38 +0200
Subject: [PATCH 0117/1225] version 0.4.390

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index d8db720e87..0e7b663204 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.365'
+__version__ = '0.4.390'

From 7040db66181d2d6d9cc8f47610a40815decdeb12 Mon Sep 17 00:00:00 2001
From: ryneeverett <ryneeverett@gmail.com>
Date: Mon, 2 May 2016 15:20:57 +0000
Subject: [PATCH 0118/1225] bandcamp: Sometimes the first key is video_caption

This fixes the exception:

    File "you-get/lib/python3.4/site-packages/you_get/extractors/bandcamp.py", line 9, in bandcamp_download
      trackinfo = json.loads(r1(r'(\[{"video_poster_url".*}\]),', html))
    File "/usr/lib64/python3.4/json/__init__.py", line 312, in loads
      s.__class__.__name__))
---
 src/you_get/extractors/bandcamp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bandcamp.py b/src/you_get/extractors/bandcamp.py
index de21a590d6..c24e4f3d88 100644
--- a/src/you_get/extractors/bandcamp.py
+++ b/src/you_get/extractors/bandcamp.py
@@ -6,7 +6,7 @@
 
 def bandcamp_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
-    trackinfo = json.loads(r1(r'(\[{"video_poster_url".*}\]),', html))
+    trackinfo = json.loads(r1(r'(\[{"(video_poster_url|video_caption)".*}\]),', html))
     for track in trackinfo:
         track_num = track['track_num']
         title = '%s. %s' % (track_num, track['title'])

From 08d65c237ca84394816ca594c58a8c84b09907df Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Tue, 3 May 2016 21:07:20 +0800
Subject: [PATCH 0119/1225] sohu: ignore invalid video format

v1, ignore invalid video format
v2, fix mistake in v1

fix: #1098

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/sohu.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index a3b4e699d9..9e06892176 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -32,9 +32,14 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
             set_proxy(tuple(extractor_proxy.split(":")))
         info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
         for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
-            hqvid = info['data'][qtyp]
+            if 'data' in info:
+                hqvid = info['data'][qtyp]
+            else:
+                hqvid = info[qtyp]
             if hqvid != 0 and hqvid != vid :
                 info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % hqvid))
+                if not 'allot' in info:
+                    continue
                 break
         if extractor_proxy:
             unset_proxy()

From a41a17fdefcca622d23a18dcf70c9d5c19c7680e Mon Sep 17 00:00:00 2001
From: natsu1211 <longyutao1211@gmail.com>
Date: Thu, 5 May 2016 16:35:09 +0900
Subject: [PATCH 0120/1225] fix url and add support to 189 video source

---
 src/you_get/extractors/tucao.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/tucao.py b/src/you_get/extractors/tucao.py
index 28e296c731..66baa3bfea 100644
--- a/src/you_get/extractors/tucao.py
+++ b/src/you_get/extractors/tucao.py
@@ -6,13 +6,14 @@
 import random
 import time
 from xml.dom import minidom
-
+#possible raw list types
 #1. <li>type=tudou&vid=199687639</li>
 #2. <li>type=tudou&vid=199506910|</li>
 #3. <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|</li>
 #4 may ? <li>type=video&file=http://xiaoshen140731.qiniudn.com/lovestage04.flv|xx**type=&vid=?</li>
 #5. <li>type=tudou&vid=200003098|07**type=tudou&vid=200000350|08</li>
-
+#6. <li>vid=49454694&type=sina|</li>
+#7. <li>type=189&vid=513031813243909|</li>
 # re_pattern=re.compile(r"(type=(.+?)&(vid|file)=(.*?))[\|<]")
 
 def tucao_single_download(type_link, title, output_dir=".", merge=True, info_only=False):
@@ -22,8 +23,17 @@ def tucao_single_download(type_link, title, output_dir=".", merge=True, info_onl
         print_info(site_info, title, vtype, size)
         if not info_only:
             download_urls([url], title, ext, size, output_dir)
+    #fix for 189 video source, see raw list types 7
+    elif "189" in type_link:
+        vid = match1(type_link, r"vid=(\d+)")
+        assert vid, "vid not exsits"
+        url = "http://api.tucao.tv/api/down/{}".format(vid)
+        vtype, ext, size=url_info(url)
+        print_info(site_info, title, vtype, size)
+        if not info_only:
+            download_urls([url], title, ext, size, output_dir)
     else:
-        u="http://www.tucao.cc/api/playurl.php?{}&key=tucao{:07x}.cc&r={}".format(type_link,random.getrandbits(28),int(time.time()*1000))
+        u="http://www.tucao.tv/api/playurl.php?{}&key=tucao{:07x}.cc&r={}".format(type_link,random.getrandbits(28),int(time.time()*1000))
         xml=minidom.parseString(get_content(u))
         urls=[]
         size=0
@@ -33,12 +43,13 @@ def tucao_single_download(type_link, title, output_dir=".", merge=True, info_onl
             size+=_size
         print_info(site_info, title, vtype, size)
         if not info_only:
-            download_urls(urls, title, ext, size, output_dir) 
+            download_urls(urls, title, ext, size, output_dir)
 
 def tucao_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
     html=get_content(url)
     title=match1(html,r'<h1 class="show_title">(.*?)<\w')
-    raw_list=match1(html,r"<li>(type=.+?)</li>")
+    #fix for raw list that vid goes before type, see raw list types 6
+    raw_list=match1(html,r"<li>\s*(type=.+?|vid=.+?)</li>")
     raw_l=raw_list.split("**")
     if len(raw_l)==1:
         format_link=raw_l[0][:-1] if raw_l[0].endswith("|") else raw_l[0]
@@ -49,6 +60,6 @@ def tucao_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
             tucao_single_download(format_link,title+"-"+sub_title,output_dir,merge,info_only)
 
 
-site_info = "tucao.cc"
+site_info = "tucao.tv"
 download = tucao_download
 download_playlist = playlist_not_supported("tucao")

From 54cafb3d3cc387b2cf2eabe2bc88d76dba173b97 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 6 May 2016 08:43:12 +0800
Subject: [PATCH 0121/1225] fix download error like this url
 http://v.qq.com/page/m/a/h/m0195j57pah.html

---
 src/you_get/extractors/qq.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 00a99cd96f..66487b8ea0 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -10,7 +10,12 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
     url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']
     fvkey = output_json['vl']['vi'][0]['fvkey']
-    url = '%s/%s.mp4?vkey=%s' % ( url, vid, fvkey )
+    mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
+    if mp4:
+        mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
+    else:
+        mp4 = output_json['vl']['vi'][0]['fn']
+    url = '%s/%s?vkey=%s' % ( url, mp4, fvkey )
     _, ext, size = url_info(url, faker=True)
 
     print_info(site_info, title, ext, size)

From b9428bd9c73aa61db4bfd4529cd38fa58b743704 Mon Sep 17 00:00:00 2001
From: Xiaobing Yu <yuxiaobing@gmail.com>
Date: Fri, 6 May 2016 18:41:11 +0800
Subject: [PATCH 0122/1225] some url like this
 http://kuaibao.qq.com/s/SPO2016050503989500 got the title failed will raise
 exception

---
 src/you_get/extractors/qq.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 66487b8ea0..ebe1a9ad3b 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -32,7 +32,8 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     elif 'kuaibao.qq.com' in url:
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
-        title = match1(content, r'title">([^"]+)</p>').strip()
+        title = match1(content, r'title">([^"]+)</p>')
+        title = title.strip() if title else vid
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From 6c3828cc32625f3caab6e82bb291a40f2416ff7f Mon Sep 17 00:00:00 2001
From: David Zhuang <i+david.zhuang-mail.utoronto.ca@cnbeining.com>
Date: Fri, 6 May 2016 16:30:08 -0400
Subject: [PATCH 0123/1225] [MangGuoTV]Add support

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/mgtv.py     | 110 +++++++++++++++++++++++++++++
 3 files changed, 112 insertions(+)
 create mode 100644 src/you_get/extractors/mgtv.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e20be32b11..284b166016 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -47,6 +47,7 @@
     'lizhi'            : 'lizhi',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
+    'mgtv'             : 'mgtv',
     'miomio'           : 'miomio',
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 5af9cdd360..6c3864fd73 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -38,6 +38,7 @@
 from .lizhi import *
 from .magisto import *
 from .metacafe import *
+from .mgtv import *
 from .miaopai import *
 from .miomio import *
 from .mixcloud import *
diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
new file mode 100644
index 0000000000..a3b7a06e3a
--- /dev/null
+++ b/src/you_get/extractors/mgtv.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+from json import loads
+from urllib.parse import urlsplit
+from os.path import dirname
+
+class MGTV(VideoExtractor):
+    name = "芒果 (MGTV)"
+
+    # Last updated: 2015-11-24
+    stream_types = [
+        {'id': 'hd', 'container': 'flv', 'video_profile': '超清'},
+        {'id': 'sd', 'container': 'flv', 'video_profile': '高清'},
+        {'id': 'ld', 'container': 'flv', 'video_profile': '标清'},
+    ]
+    
+    id_dic = {i['video_profile']:(i['id']) for i in stream_types}
+    
+    api_endpoint = 'http://v.api.mgtv.com/player/video?video_id={video_id}'
+
+    @staticmethod
+    def get_vid_from_url(url):
+        """Extracts video ID from URL.
+        """
+        return match1(url, 'http://www.mgtv.com/v/\d/\d+/\w+/(\d+).html')
+    
+    #----------------------------------------------------------------------
+    @staticmethod
+    def get_mgtv_real_url(url):
+        """str->list of str
+        Give you the real URLs."""
+        content = loads(get_content(url))
+        m3u_url = content['info']
+        split = urlsplit(m3u_url)
+        
+        base_url = "{scheme}://{netloc}{path}/".format(scheme = split[0],
+                                                      netloc = split[1],
+                                                      path = dirname(split[2]))
+
+        content = get_content(content['info'])  #get the REAL M3U url, maybe to be changed later?
+        segment_list = []
+        for i in content.split():
+            if not i.startswith('#'):  #not the best way, better we use the m3u8 package
+                segment_list.append(base_url + i)
+        return segment_list
+
+    def download_playlist_by_url(self, url, **kwargs):
+        pass
+
+    def prepare(self, **kwargs):
+        if self.url:
+            self.vid = self.get_vid_from_url(self.url)
+        content = get_content(self.api_endpoint.format(video_id = self.vid))
+        content = loads(content)
+        self.title = content['data']['info']['title']
+        
+        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        stream_available = {}
+        for i in content['data']['stream']:
+            stream_available[i['name']] = i['url']
+
+        for s in self.stream_types:
+            if s['video_profile'] in stream_available.keys():
+                quality_id = self.id_dic[s['video_profile']]
+                url = stream_available[s['video_profile']]
+                segment_list_this = self.get_mgtv_real_url(url)
+                
+                container_this_stream = ''
+                size_this_stream = 0
+                stream_fileid_list = []
+                for i in segment_list_this:
+                    _, container_this_stream, size_this_seg = url_info(i)
+                    size_this_stream += size_this_seg
+                    stream_fileid_list.append(os.path.basename(i).split('.')[0])
+                    
+            #make pieces
+            pieces = []
+            for i in zip(stream_fileid_list, segment_list_this):
+                pieces.append({'fileid': i[0], 'segs': i[1],})
+
+                self.streams[quality_id] = {
+                        'container': 'flv',
+                        'video_profile': s['video_profile'],
+                        'size': size_this_stream,
+                        'pieces': pieces
+                    }
+                
+            if not kwargs['info_only']:
+                self.streams[quality_id]['src'] = segment_list_this
+
+    def extract(self, **kwargs):
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+
+site = MGTV()
+download = site.download_by_url
+download_playlist = site.download_playlist_by_url
\ No newline at end of file

From 23b3a9040b4dad0e74841398aaedbc83ac2106d5 Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 00:23:16 +0800
Subject: [PATCH 0124/1225] match 'http://www.dilidili.com/watch' and '/watch3'

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 6bc6ae2831..f3f8e197ac 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch/\w+', url):
+    if re.match(r'http://www.dilidili.com/watch\w+', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From 17533bf6933d3c33be2c238c987275dd1c6da0b4 Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 22:18:34 +0800
Subject: [PATCH 0125/1225] fix the url 'watch/' and 'watch3/' error

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index f3f8e197ac..8433294bbd 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch\w+', url):
+    if re.match(r'http://www.dilidili.com/watch\S', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From f5c22cdfef99878027f3f9e78ff9ae2701a21a3f Mon Sep 17 00:00:00 2001
From: ksnzh <ksnzher@gmail.com>
Date: Tue, 10 May 2016 23:04:55 +0800
Subject: [PATCH 0126/1225] fix the url 'watch/' and 'watch3/' error

---
 src/you_get/extractors/dilidili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 8433294bbd..082f84e130 100755
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -35,7 +35,7 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch\S', url):
+    if re.match(r'http://www.dilidili.com/watch\S+', url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         

From e697287b95e1ddee50047c34acd2f8f9a347e1eb Mon Sep 17 00:00:00 2001
From: iliul <liul.stone@gmail.com>
Date: Wed, 11 May 2016 14:28:50 +0800
Subject: [PATCH 0127/1225] add support Panda.tv

---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/panda.py    | 32 ++++++++++++++++++++++++++++++
 4 files changed, 35 insertions(+)
 create mode 100644 src/you_get/extractors/panda.py

diff --git a/README.md b/README.md
index ddfac26d35..a4f4fcd996 100644
--- a/README.md
+++ b/README.md
@@ -344,6 +344,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Dilidili | <http://www.dilidili.com/>     |✓| | |
 | 豆瓣     | <http://www.douban.com/>       | | |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
+| Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
 | 风行网   | <http://www.fun.tv/>           |✓| | |
 | iQIYI<br/>爱奇艺 | <http://www.iqiyi.com/> |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 284b166016..ab32a62df1 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -54,6 +54,7 @@
     'musicplayon'      : 'musicplayon',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
+    'panda'            : 'panda',
     'pinterest'        : 'pinterest',
     'pixnet'           : 'pixnet',
     'pptv'             : 'pptv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 6c3864fd73..1bb7a7ab16 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -47,6 +47,7 @@
 from .nanagogo import *
 from .netease import *
 from .nicovideo import *
+from .panda import *
 from .pinterest import *
 from .pixnet import *
 from .pptv import *
diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
new file mode 100644
index 0000000000..74064f9fcf
--- /dev/null
+++ b/src/you_get/extractors/panda.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+__all__ = ['panda_download']
+
+from ..common import *
+import json
+import time
+
+def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    roomid = url[url.rfind('/')+1:]
+    json_request_url = 'http://www.panda.tv/api_room?roomid={}&pub_key=&_={}'.format(roomid, int(time.time()))
+    content = get_html(json_request_url)
+    errno = json.loads(content)['errno']
+    errmsg = json.loads(content)['errmsg']
+    if errno:
+        raise ValueError("Errno : {}, Errmsg : {}".format(errno, errmsg))
+
+    data = json.loads(content)['data']
+    title = data.get('roominfo')['name']
+    room_key = data.get('videoinfo')['room_key']
+    status = data.get('videoinfo')['status']
+    if status is not "2":
+        raise ValueError("The live stream is not online! (status:%s)" % status)
+    real_url = 'http://pl3.live.panda.tv/live_panda/{}.flv'.format(room_key)
+
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+
+site_info = "panda.tv"
+download = panda_download
+download_playlist = playlist_not_supported('panda')

From 03a4d3cc91c91dcc6908b070ae98ac68100c2669 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Thu, 12 May 2016 23:56:34 -0700
Subject: [PATCH 0128/1225] common.py: Print merged filename after successful
 merge

Instead of just "Done.".
---
 src/you_get/common.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 284b166016..f35cdf1863 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -735,7 +735,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
             if has_ffmpeg_installed():
                 from .processor.ffmpeg import ffmpeg_concat_av
                 ret = ffmpeg_concat_av(parts, output_filepath, ext)
-                print('Done.')
+                print('Merged into %s' % output_filename)
                 if ret == 0:
                     for part in parts: os.remove(part)
 
@@ -748,7 +748,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_flv import concat_flv
                     concat_flv(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:
@@ -764,7 +764,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_mp4 import concat_mp4
                     concat_mp4(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:
@@ -780,7 +780,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 else:
                     from .processor.join_ts import concat_ts
                     concat_ts(parts, output_filepath)
-                print('Done.')
+                print('Merged into %s' % output_filename)
             except:
                 raise
             else:

From 4f91f908dbb24b19db65659afc6e37b8bcddcc10 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 14 May 2016 23:36:51 +0200
Subject: [PATCH 0129/1225] [yinyuetai] fix #1102

---
 src/you_get/extractors/yinyuetai.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/yinyuetai.py b/src/you_get/extractors/yinyuetai.py
index dc4a9364ea..6c39540fe1 100644
--- a/src/you_get/extractors/yinyuetai.py
+++ b/src/you_get/extractors/yinyuetai.py
@@ -17,7 +17,8 @@ def yinyuetai_download_by_id(vid, title=None, output_dir='.', merge=True, info_o
         download_urls([url], title, ext, size, output_dir, merge = merge)
 
 def yinyuetai_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    id = r1(r'http://\w+.yinyuetai.com/video/(\d+)', url)
+    id = r1(r'http://\w+.yinyuetai.com/video/(\d+)', url) or \
+         r1(r'http://\w+.yinyuetai.com/video/h5/(\d+)', url)
     if not id:
         yinyuetai_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
         return

From f7c20b236c967e6241b48a430c14a6e97681cefd Mon Sep 17 00:00:00 2001
From: David Zhuang <Beining@Davids-MacBook-Pro.local>
Date: Tue, 17 May 2016 15:28:33 -0400
Subject: [PATCH 0130/1225] [MGTV]Fix some video cannot be downloaded in full
 length

---
 src/you_get/extractors/mgtv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index a3b7a06e3a..aeb4249072 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -7,6 +7,7 @@
 from json import loads
 from urllib.parse import urlsplit
 from os.path import dirname
+import re
 
 class MGTV(VideoExtractor):
     name = "芒果 (MGTV)"
@@ -67,6 +68,7 @@ def prepare(self, **kwargs):
             if s['video_profile'] in stream_available.keys():
                 quality_id = self.id_dic[s['video_profile']]
                 url = stream_available[s['video_profile']]
+                url = re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
                 segment_list_this = self.get_mgtv_real_url(url)
                 
                 container_this_stream = ''

From 677040a39ef448f8ccf3f4c8056642b6a9a9d585 Mon Sep 17 00:00:00 2001
From: NickeyKim <mmf760@gmail.com>
Date: Thu, 19 May 2016 14:48:45 +0900
Subject: [PATCH 0131/1225] add Naver support

add Naver support
---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/naver.py    | 48 ++++++++++++++++++++++++++++++
 4 files changed, 51 insertions(+)
 create mode 100644 src/you_get/extractors/naver.py

diff --git a/README.md b/README.md
index a4f4fcd996..d0a91aa88f 100644
--- a/README.md
+++ b/README.md
@@ -373,6 +373,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
 | 花瓣     | <http://huaban.com/>           | |✓| |
+| Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6c65bd4936..9668df5886 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -52,6 +52,7 @@
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
     'musicplayon'      : 'musicplayon',
+    'naver'            : 'naver',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
     'panda'            : 'panda',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 1bb7a7ab16..6fdaa340b1 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -45,6 +45,7 @@
 from .mtv81 import *
 from .musicplayon import *
 from .nanagogo import *
+from .naver import *
 from .netease import *
 from .nicovideo import *
 from .panda import *
diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
new file mode 100644
index 0000000000..7f7d554875
--- /dev/null
+++ b/src/you_get/extractors/naver.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+__all__ = ['naver_download']
+import urllib.request, urllib.parse
+from ..common import *
+
+def naver_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+
+	assert re.search(r'http://tvcast.naver.com/v/', url), "URL is not supported"
+
+	html = get_html(url)
+	contentid = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',html)
+	videoid = contentid.group(1)
+	inkey = contentid.group(2)
+	assert videoid
+	assert inkey
+	info_key = urllib.parse.urlencode({'vid': videoid, 'inKey': inkey, })
+	down_key = urllib.parse.urlencode({'masterVid': videoid,'protocol': 'p2p','inKey': inkey, })
+	inf_xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?%s' % info_key )
+
+	from xml.dom.minidom import parseString
+	doc_info = parseString(inf_xml)
+	Subject = doc_info.getElementsByTagName('Subject')[0].firstChild
+	title = Subject.data
+	assert title
+
+	xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?%s' % down_key )
+	doc = parseString(xml)
+
+	encodingoptions = doc.getElementsByTagName('EncodingOption')
+	old_height = doc.getElementsByTagName('height')[0]
+	real_url= ''
+	#to download the highest resolution one,
+	for node in encodingoptions:
+		new_height = node.getElementsByTagName('height')[0]
+		domain_node = node.getElementsByTagName('Domain')[0]
+		uri_node = node.getElementsByTagName('uri')[0]
+		if int(new_height.firstChild.data) > int (old_height.firstChild.data):
+			real_url= domain_node.firstChild.data+ '/' +uri_node.firstChild.data
+
+	type, ext, size = url_info(real_url)
+	print_info(site_info, title, type, size)
+	if not info_only:
+		download_urls([real_url], title, ext, size, output_dir, merge = merge)
+
+site_info = "tvcast.naver.com"
+download = naver_download
+download_playlist = playlist_not_supported('naver')

From e6af1c6265be5a45f978f3321db20cfe9cb30a05 Mon Sep 17 00:00:00 2001
From: jmargeta <jmargeta@gmail.com>
Date: Thu, 19 May 2016 22:57:48 +0200
Subject: [PATCH 0132/1225] Support for embedded Vimeo videos

Example URL:
- http://www.miracletutorials.com/howto-embed-vimeo/
---
 src/you_get/extractors/embed.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index fd463c92bc..a177e66394 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -8,6 +8,7 @@
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
+from .vimeo import vimeo_download_by_id
 from .yinyuetai import yinyuetai_download_by_id
 from .youku import youku_download_by_vid
 
@@ -39,6 +40,9 @@
 
 netease_embed_patterns = [ '(http://\w+\.163\.com/movie/[^\'"]+)' ]
 
+vimeo_embed_patters = [ 'player\.vimeo\.com/video/(\d+)' ]
+
+
 def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
     content = get_content(url, headers=fake_headers)
     found = False
@@ -69,6 +73,11 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         netease_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
+    urls = matchall(content, vimeo_embed_patters)
+    for url in urls:
+        found = True
+        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+
     if not found:
         raise NotImplementedError(url)
 

From 2cd4656b32c1005edebf3047ba7f1f7470abfbf0 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 20 May 2016 04:28:30 -0400
Subject: [PATCH 0133/1225] [QiE]Add Support

---
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/qie.py      | 78 ++++++++++++++++++++++++++++++
 src/you_get/extractors/qq.py       |  4 ++
 3 files changed, 83 insertions(+)
 create mode 100644 src/you_get/extractors/qie.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 1bb7a7ab16..e2cf656c16 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -52,6 +52,7 @@
 from .pixnet import *
 from .pptv import *
 from .qianmo import *
+from .qie import *
 from .qq import *
 from .sina import *
 from .sohu import *
diff --git a/src/you_get/extractors/qie.py b/src/you_get/extractors/qie.py
new file mode 100644
index 0000000000..2288106a09
--- /dev/null
+++ b/src/you_get/extractors/qie.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+from json import loads
+
+class QiE(VideoExtractor):
+    name = "QiE （企鹅直播）"
+
+    # Last updated: 2015-11-24
+    stream_types = [
+        {'id': 'normal', 'container': 'flv', 'video_profile': '标清'},
+        {'id': 'middle', 'container': 'flv', 'video_profile': '550'},
+        {'id': 'middle2', 'container': 'flv', 'video_profile': '900'},
+    ]
+    
+    id_dic = {i['video_profile']:(i['id']) for i in stream_types}
+    
+    api_endpoint = 'http://www.qie.tv/api/v1/room/{room_id}'
+
+    @staticmethod
+    def get_vid_from_url(url):
+        """Extracts video ID from live.qq.com.
+        """
+        html = get_content(url)
+        return match1(html, r'room_id\":(\d+)')
+
+    def download_playlist_by_url(self, url, **kwargs):
+        pass
+
+    def prepare(self, **kwargs):
+        if self.url:
+            self.vid = self.get_vid_from_url(self.url)
+        
+        content = get_content(self.api_endpoint.format(room_id = self.vid))
+        content = loads(content)
+        self.title = content['data']['room_name']
+        rtmp_url =  content['data']['rtmp_url']
+        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        stream_available = {}
+        stream_available['normal'] = rtmp_url + '/' + content['data']['rtmp_live']
+        if len(content['data']['rtmp_multi_bitrate']) > 0:
+            for k , v in content['data']['rtmp_multi_bitrate'].items():
+                stream_available[k] = rtmp_url + '/' + v
+        
+        for s in self.stream_types:
+            if s['id'] in stream_available.keys():
+                quality_id = s['id']
+                url = stream_available[quality_id]
+                self.streams[quality_id] = {
+                    'container': 'flv',
+                    'video_profile': s['video_profile'],
+                    'size': 0,
+                    'url': url
+                }
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            s['src'] = [s['url']]
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+            s['src'] = [s['url']]
+
+site = QiE()
+download = site.download_by_url
+download_playlist = playlist_not_supported('QiE')
\ No newline at end of file
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index ebe1a9ad3b..cb4aeebf0e 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -3,6 +3,7 @@
 __all__ = ['qq_download']
 
 from ..common import *
+from .qie import download as qieDownload
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
@@ -34,6 +35,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
+    elif 'live.qq.com' in url:
+        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
+        exit()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is

From b4eb73965ccadd9ba78e9d65d1c05b93c9467979 Mon Sep 17 00:00:00 2001
From: cnbeining <cnbeining@gmail.com>
Date: Thu, 7 Apr 2016 16:43:17 -0400
Subject: [PATCH 0134/1225] Add -bsf:a aac_adtstoasc when merging

As in #813.
---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 mode change 100644 => 100755 src/you_get/processor/ffmpeg.py

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100644
new mode 100755
index 320eb6421c..e7ee35d6b7
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -169,7 +169,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
 
         params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
         params.append(output + '.txt')
-        params += ['-c', 'copy', output]
+        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', output]
 
         subprocess.check_call(params)
         os.remove(output + '.txt')

From 67d18c766ae832864ad8188e71f981db24983025 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 22 May 2016 17:28:01 +0200
Subject: [PATCH 0135/1225] processor/ffmpeg.py: 644

---
 src/you_get/processor/ffmpeg.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/processor/ffmpeg.py

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100755
new mode 100644

From d1d62ae3040c98b7c3efb81937db211480ecb859 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 17:28:29 +0200
Subject: [PATCH 0136/1225] [twitter] fix #1139

---
 src/you_get/extractors/twitter.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 251cb8d53b..1794081b4c 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -5,6 +5,13 @@
 from ..common import *
 from .vine import vine_download
 
+def extract_m3u(source):
+    r1 = get_content(source)
+    s1 = re.findall(r'(/ext_tw_video/.*)', r1)
+    r2 = get_content('https://video.twimg.com%s' % s1[-1])
+    s2 = re.findall(r'(/ext_tw_video/.*)', r2)
+    return 'https://video.twimg.com%s' % s2[-1]
+
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
     screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
@@ -63,6 +70,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
 
+        source = extract_m3u(source)
         mime, ext, size = url_info(source)
 
         print_info(site_info, page_title, mime, size)

From d2d6fcb95397b969272a53dc9abf852f1181916d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:13:12 +0200
Subject: [PATCH 0137/1225] [twitter] fix #1139 (really)

---
 src/you_get/extractors/twitter.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 1794081b4c..b0c891891b 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -10,7 +10,7 @@ def extract_m3u(source):
     s1 = re.findall(r'(/ext_tw_video/.*)', r1)
     r2 = get_content('https://video.twimg.com%s' % s1[-1])
     s2 = re.findall(r'(/ext_tw_video/.*)', r2)
-    return 'https://video.twimg.com%s' % s2[-1]
+    return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
@@ -70,12 +70,13 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
 
-        source = extract_m3u(source)
-        mime, ext, size = url_info(source)
+        urls = extract_m3u(source)
+        size = urls_size(urls)
+        mime, ext = 'video/mp4', 'mp4'
 
         print_info(site_info, page_title, mime, size)
         if not info_only:
-            download_urls([source], page_title, ext, size, output_dir, merge=merge)
+            download_urls(urls, page_title, ext, size, output_dir, merge=merge)
 
 site_info = "Twitter.com"
 download = twitter_download

From 186762b4b51679d1af40139e782e2b722a52c633 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:26:36 +0200
Subject: [PATCH 0138/1225] update README

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index d0a91aa88f..60cb125a75 100644
--- a/README.md
+++ b/README.md
@@ -360,6 +360,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | PPTV聚力 | <http://www.pptv.com/>         |✓| | |
 | 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
 | QQ<br/>腾讯视频 | <http://v.qq.com/>      |✓| | |
+| 企鹅直播 | <http://live.qq.com/>          |✓| | |
 | 阡陌视频 | <http://qianmo.com/>           |✓| | |
 | THVideo  | <http://thvideo.tv/>           |✓| | |
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
@@ -374,6 +375,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
 | 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
+| 芒果TV   | <http://www.mgtv.com/>         |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 

From 5f99e1dfd3ca5d6c83b11e5c060a8c0bfe611d8d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 May 2016 22:38:19 +0200
Subject: [PATCH 0139/1225] version 0.4.424

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 0e7b663204..5a9d5581bc 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.390'
+__version__ = '0.4.424'

From 854386f22fd7c38462e3d89f46c94f5c13ac197b Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Sun, 22 May 2016 23:07:10 -0700
Subject: [PATCH 0140/1225] Retry request if timeout in get_content

---
 src/you_get/common.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9668df5886..dadba69c75 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -98,6 +98,7 @@
 import os
 import platform
 import re
+import socket
 import sys
 import time
 from urllib import request, parse, error
@@ -308,7 +309,14 @@ def get_content(url, headers={}, decoded=True):
     if cookies:
         cookies.add_cookie_header(req)
         req.headers.update(req.unredirected_hdrs)
-    response = request.urlopen(req)
+
+    for i in range(10):
+        try:
+            response = request.urlopen(req, timeout=10)
+            break
+        except socket.timeout:
+            logging.debug('request attempt %s timeout' % str(i + 1))
+
     data = response.read()
 
     # Handle HTTP compression for gzip and deflate (zlib)

From 48640e279f451cb2d365154870ed20e2d8bcc3e4 Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Tue, 24 May 2016 20:58:28 -0700
Subject: [PATCH 0141/1225] Add -t and --timeout option

To set socket timeout, default is 600 seconds.
---
 src/you_get/common.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index dadba69c75..100f3869d7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -312,7 +312,7 @@ def get_content(url, headers={}, decoded=True):
 
     for i in range(10):
         try:
-            response = request.urlopen(req, timeout=10)
+            response = request.urlopen(req)
             break
         except socket.timeout:
             logging.debug('request attempt %s timeout' % str(i + 1))
@@ -1071,11 +1071,12 @@ def version():
     -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
          --no-proxy                     Never use a proxy.
+    -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=']
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:t:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -1105,6 +1106,7 @@ def version():
     proxy = None
     extractor_proxy = None
     traceback = False
+    timeout = 600
     for o, a in opts:
         if o in ('-V', '--version'):
             version()
@@ -1178,6 +1180,8 @@ def version():
             extractor_proxy = a
         elif o in ('--lang',):
             lang = a
+        elif o in ('-t', '--timeout'):
+            timeout = int(a)
         else:
             log.e("try 'you-get --help' for more options")
             sys.exit(2)
@@ -1187,6 +1191,8 @@ def version():
 
     set_http_proxy(proxy)
 
+    socket.setdefaulttimeout(timeout)
+
     try:
         if stream_id:
             if not extractor_proxy:

From c89c977189372c64978b9d8aac0906047c6affde Mon Sep 17 00:00:00 2001
From: osinx zhou <osinx.zhou@gmail.com>
Date: Wed, 25 May 2016 15:35:41 +0800
Subject: [PATCH 0142/1225]  Support http://baidu.ku6.com #1163

---
 src/you_get/extractors/ku6.py | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index d9a1ef1280..7f28c75bcc 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -27,13 +27,30 @@ def ku6_download_by_id(id, title = None, output_dir = '.', merge = True, info_on
         download_urls(urls, title, ext, size, output_dir, merge = merge)
 
 def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    patterns = [r'http://v.ku6.com/special/show_\d+/(.*)\.\.\.html',
-            r'http://v.ku6.com/show/(.*)\.\.\.html',
-            r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
-    id = r1_of(patterns, url)
+    id = None
+
+    if match1(url, r'http://baidu.ku6.com/watch/(.*)\.html') is not None:
+        id = baidu_ku6(url)
+    else:
+        patterns = [r'http://v.ku6.com/special/show_\d+/(.*)\.\.\.html',
+                r'http://v.ku6.com/show/(.*)\.\.\.html',
+                r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
+        id = r1_of(patterns, url)
 
     ku6_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
 
+def baidu_ku6(url):
+    id = None
+
+    h1 = get_html(url)
+    isrc = match1(h1, r'<iframe id="innerFrame" src="([^"]*)"')
+
+    if isrc is not None:
+        h2 = get_html(isrc)
+        id = match1(h2, r'http://v.ku6.com/show/(.*)\.\.\.html')
+
+    return id
+
 site_info = "Ku6.com"
 download = ku6_download
 download_playlist = playlist_not_supported('ku6')

From b6a00c3e14114755f4c92dc9eb603e3ff621bf45 Mon Sep 17 00:00:00 2001
From: 4ft35t <4ft35t@gmail.com>
Date: Thu, 26 May 2016 23:52:22 +0800
Subject: [PATCH 0143/1225] fix path error

---
 you-get | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/you-get b/you-get
index 85f3f754b7..8529388f98 100755
--- a/you-get
+++ b/you-get
@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import os, sys
 
-_srcdir = 'src/'
+_srcdir = '%s/src/' % os.path.dirname(os.path.realpath(__file__))
 _filepath = os.path.dirname(sys.argv[0])
 sys.path.insert(1, os.path.join(_filepath, _srcdir))
 

From 8879bca209c4655173134faad2a2dd60ea87ebee Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 May 2016 01:48:51 +0200
Subject: [PATCH 0144/1225] [videomega] fix support

---
 src/you_get/extractors/videomega.py | 78 +++++++++++++----------------
 1 file changed, 36 insertions(+), 42 deletions(-)

diff --git a/src/you_get/extractors/videomega.py b/src/you_get/extractors/videomega.py
index 75e88cd97d..a2965e7a55 100644
--- a/src/you_get/extractors/videomega.py
+++ b/src/you_get/extractors/videomega.py
@@ -1,47 +1,41 @@
 #!/usr/bin/env python
 
-from ..common import *
-from ..extractor import VideoExtractor
+__all__ = ['videomega_download']
 
+from ..common import *
 import ssl
 
-class Videomega(VideoExtractor):
-    name = "Videomega"
-
-    stream_types = [
-        {'id': 'original'}
-    ]
-
-    def prepare(self, **kwargs):
-        # Hot-plug cookie handler
-        ssl_context = request.HTTPSHandler(
-            context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
-        cookie_handler = request.HTTPCookieProcessor()
-        opener = request.build_opener(ssl_context, cookie_handler)
-        opener.addheaders = [('Referer', self.url),
-                             ('Cookie', 'noadvtday=0')]
-        request.install_opener(opener)
-
-        ref = match1(self.url, r'ref=(\w+)')
-        php_url = 'http://videomega.tv/view.php?ref=' + ref
-        content = get_content(php_url)
-
-        self.title = match1(content, r'<title>(.*)</title>')
-        js = match1(content, r'(eval.*)')
-        t = match1(js, r'\$\("\d+"\)\.\d+\("\d+","([^"]+)"\)')
-        t = re.sub(r'(\w)', r'{\1}', t)
-        t = t.translate({87 + i: str(i) for i in range(10, 36)})
-        s = match1(js, r"'([^']+)'\.split").split('|')
-        self.streams['original'] = {
-            'url': t.format(*s)
-        }
-
-    def extract(self, **kwargs):
-        for i in self.streams:
-            s = self.streams[i]
-            _, s['container'], s['size'] = url_info(s['url'])
-            s['src'] = [s['url']]
-
-site = Videomega()
-download = site.download_by_url
-download_playlist = site.download_by_url
+def videomega_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    # Hot-plug cookie handler
+    ssl_context = request.HTTPSHandler(
+        context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
+    cookie_handler = request.HTTPCookieProcessor()
+    opener = request.build_opener(ssl_context, cookie_handler)
+    opener.addheaders = [('Referer', url),
+                         ('Cookie', 'noadvtday=0')]
+    request.install_opener(opener)
+
+    content = get_content(url)
+    m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
+    ref = m.group(1)
+    width, height = m.group(2), m.group(3)
+    php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
+    content = get_content(php_url)
+
+    title = match1(content, r'<title>(.*)</title>')
+    js = match1(content, r'(eval.*)')
+    t = match1(js, r'\$\("\w+"\)\.\w+\("\w+","([^"]+)"\)')
+    t = re.sub(r'(\w)', r'{\1}', t)
+    t = t.translate({87 + i: str(i) for i in range(10, 36)})
+    s = match1(js, r"'([^']+)'\.split").split('|')
+    src = t.format(*s)
+
+    type, ext, size = url_info(src, faker=True)
+
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([src], title, ext, size, output_dir, merge=merge, faker=True)
+
+site_info = "Videomega.tv"
+download = videomega_download
+download_playlist = playlist_not_supported('videomega')

From 71ec9cad223f4ee2022d2530c2bc5154213907ef Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 May 2016 03:12:49 +0200
Subject: [PATCH 0145/1225] [videomega] support embedded URLs

---
 src/you_get/extractors/videomega.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/videomega.py b/src/you_get/extractors/videomega.py
index a2965e7a55..34fb5205a1 100644
--- a/src/you_get/extractors/videomega.py
+++ b/src/you_get/extractors/videomega.py
@@ -15,11 +15,14 @@ def videomega_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                          ('Cookie', 'noadvtday=0')]
     request.install_opener(opener)
 
-    content = get_content(url)
-    m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
-    ref = m.group(1)
-    width, height = m.group(2), m.group(3)
-    php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
+    if re.search(r'view\.php', url):
+        php_url = url
+    else:
+        content = get_content(url)
+        m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
+        ref = m.group(1)
+        width, height = m.group(2), m.group(3)
+        php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
     content = get_content(php_url)
 
     title = match1(content, r'<title>(.*)</title>')

From 9c67f4025861ac7a4b19562b0255c3e0c3b6f5e1 Mon Sep 17 00:00:00 2001
From: jackyzy823 <jackyzy823@gmail.com>
Date: Fri, 27 May 2016 16:18:56 +0000
Subject: [PATCH 0146/1225] [Netease] add support for radio & program

---
 src/you_get/extractors/netease.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 1fc7c07171..332d48ec1d 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -61,7 +61,7 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
                 netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
             except: pass
 
-    elif "song" in url:
+    elif "song" in url: 
         j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
         try: # download lyrics
@@ -70,6 +70,15 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
             netease_lyric_download(j["songs"][0], l["lrc"]["lyric"], output_dir=output_dir, info_only=info_only)
         except: pass
 
+    elif "program" in url:
+        j = loads(get_content("http://music.163.com/api/dj/program/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
+        netease_song_download(j["program"]["mainSong"], output_dir=output_dir, info_only=info_only)
+
+    elif "radio" in url:
+        j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
+        for i in j['programs']:
+            netease_song_download(i["mainSong"],output_dir=output_dir, info_only=info_only)
+
     elif "mv" in url:
         j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_video_download(j['data'], output_dir=output_dir, info_only=info_only)

From b9431422d8cdd0e74c55bbaf02b79d1f75c16bd8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 May 2016 04:19:25 +0200
Subject: [PATCH 0147/1225] [twitter] support scribe_playlist_url (vine)

---
 src/you_get/extractors/twitter.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index b0c891891b..09956141b9 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -69,8 +69,14 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             vmap = get_content(vmap_url)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
             if not item_id: page_title = i['tweet_id']
+        elif 'scribe_playlist_url' in i:
+            scribe_playlist_url = i['scribe_playlist_url']
+            return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)
 
-        urls = extract_m3u(source)
+        if source.endswith('.mp4'):
+            urls = [source]
+        else:
+            urls = extract_m3u(source)
         size = urls_size(urls)
         mime, ext = 'video/mp4', 'mp4'
 

From fd2a290ffe9193043b025ceb122042cf6623931d Mon Sep 17 00:00:00 2001
From: jackyzy823 <jackyzy823@gmail.com>
Date: Sat, 28 May 2016 09:32:07 +0000
Subject: [PATCH 0148/1225] [QQ] more robust url extractor

---
 src/you_get/extractors/qq.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index cb4aeebf0e..783bb2f5a5 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -4,7 +4,7 @@
 
 from ..common import *
 from .qie import download as qieDownload
-
+from urllib.parse import urlparse,parse_qs
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
@@ -24,31 +24,35 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if 'live.qq.com' in url:
+        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
+        return 
+
+    #do redirect
     if 'v.qq.com/page' in url:
         # for URLs like this:
         # http://v.qq.com/page/k/9/7/k0194pwgw97.html
-        # it will redirect.
-        vid = match1(url, r'\b(\w+).html')
-        title = vid
-    elif 'kuaibao.qq.com' in url:
+        content = get_html(url)
+        url = match1(content,r'window\.location\.href="(.*?)"')
+        
+    if 'kuaibao.qq.com' in url:
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
-    elif 'live.qq.com' in url:
-        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
-        exit()
     elif 'iframe/player.html' in url:
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is
         title = vid
     else:
         content = get_html(url)
-        vid = match1(content, r'vid\s*:\s*"\s*([^"]+)"')
-        title = match1(content, r'title\s*:\s*"\s*([^"]+)"')
-        # try to get the right title for URLs like this:
-        # http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        title = matchall(content, [r'title\s*:\s*"\s*([^"]+)"'])[-1]
+        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5 
+        vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
+        title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
+        title = match1(content, r'title">([^"]+)</p>') if not title else title
+        title = vid if not title else title #general fallback
+
+
 
     qq_download_by_vid(vid, title, output_dir, merge, info_only)
 

From 10d6566552a621bc74e9e3d4e3b695e3797892dd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 May 2016 16:11:49 +0200
Subject: [PATCH 0149/1225] [bilibili] fix #1176

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1a6ca325dc..55293533fa 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -11,7 +11,7 @@
 import hashlib
 import re
 
-appkey='8e9fc618fbd41e28'
+appkey='f3bb208b3d081dc8'
 
 def get_srt_xml(id):
     url = 'http://comment.bilibili.com/%s.xml' % id

From 3b4a576e6cefa31f0bb25ec91f52814563384502 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Sat, 28 May 2016 18:50:18 -0700
Subject: [PATCH 0150/1225] Add shell completion definitions for Bash, Fish and
 Zsh

Paths of completion definitions in this repo are inconsequential, and I
don't know a convention for that, so I just modelled it on git.git
https://github.com/git/git/tree/master/contrib/completion. What's
different here is that instead of using a uniform naming scheme for each
file (you-get-completion.<shellname>), I'm using the conventional name
for each shell, which means _you-get for Zsh, for instance. Users of
each shell should be able to pick out the completion definition for
their shell based on the filenames; in the worst case, they can still
tell from the comment at the top of each file.

Note that I'm no expert on Bash or Fish completion, so
you-get-completion.bash and you-get.fish may not be written in the
idiomatic way. (you-get-completion.bash is most certainly not idiomatic,
if there's an idiom at all.)

Closes #1178.
---
 README.md                                  |  4 +++
 contrib/completion/_you-get                | 29 ++++++++++++++++++++
 contrib/completion/you-get-completion.bash | 31 ++++++++++++++++++++++
 contrib/completion/you-get.fish            | 23 ++++++++++++++++
 4 files changed, 87 insertions(+)
 create mode 100644 contrib/completion/_you-get
 create mode 100755 contrib/completion/you-get-completion.bash
 create mode 100644 contrib/completion/you-get.fish

diff --git a/README.md b/README.md
index 60cb125a75..af2cdf9dd5 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,10 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Shell completion
+
+Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](contrib/completion). Please consult your shell's manual for how to take advantage of them.
+
 ## Upgrading
 
 Based on which option you chose to install `you-get`, you may upgrade it via:
diff --git a/contrib/completion/_you-get b/contrib/completion/_you-get
new file mode 100644
index 0000000000..696aad891c
--- /dev/null
+++ b/contrib/completion/_you-get
@@ -0,0 +1,29 @@
+#compdef you-get
+
+# Zsh completion definition for soimort/you-get.
+
+setopt localoptions noshwordsplit noksharrays
+local -a args
+
+args=(
+    '(- : *)'{-V,--version}'[print version and exit]'
+    '(- : *)'{-h,--help}'[print help and exit]'
+    '(-i --info)'{-i,--info}'[print extracted information]'
+    '(-u --url)'{-u,--url}'[print extracted information with URLs]'
+    '(--json)--json[print extracted URLs in JSON format]'
+    '(-n --no-merge)'{-n,--no-merge}'[do not merge video parts]'
+    '(--no-caption)--no-caption[do not download captions]'
+    '(-f --force)'{-f,--force}'[force overwrite existing files]'
+    '(-F --format)'{-F,--format}'[set video format to the specified stream id]:stream id'
+    '(-O --output-filename)'{-O,--output-filename}'[set output filename]:filename:_files'
+    '(-o --output-dir)'{-o,--output-dir}'[set output directory]:directory:_files -/'
+    '(-p --player)'{-p,--player}'[stream extracted URL to the specified player]:player and options'
+    '(-c --cookies)'{-c,--cookies}'[load cookies.txt or cookies.sqlite]:cookies file:_files'
+    '(-x --http-proxy)'{-x,--http-proxy}'[use the specified HTTP proxy for downloading]:host\:port:'
+    '(-y --extractor-proxy)'{-y,--extractor-proxy}'[use the specified HTTP proxy for extraction only]:host\:port'
+    '(--no-proxy)--no-proxy[do not use a proxy]'
+    '(-t --timeout)'{-t,--timeout}'[set socket timeout]:seconds'
+    '(-d --debug)'{-d,--debug}'[show traceback and other debug info]'
+    '*: :_guard "^-*" url'
+)
+_arguments -S -s $args
diff --git a/contrib/completion/you-get-completion.bash b/contrib/completion/you-get-completion.bash
new file mode 100755
index 0000000000..9c6480ec6b
--- /dev/null
+++ b/contrib/completion/you-get-completion.bash
@@ -0,0 +1,31 @@
+# Bash completion definition for you-get.
+
+_you-get () {
+    COMPREPLY=()
+    local IFS=$' \n'
+    local cur=$2 prev=$3
+    local -a opts_without_arg opts_with_arg
+    opts_without_arg=(
+        -V --version -h --help -i --info -u --url --json -n --no-merge
+        --no-caption -f --force --no-proxy -d --debug
+    )
+    opts_with_arg=(
+        -F --format -O --output-filename -o --output-dir -p --player
+        -c --cookies -x --http-proxy -y --extractor-proxy -t --timeout
+    )
+
+    # Do not complete non option names
+    [[ $cur == -* ]] || return 1
+
+    # Do not complete when the previous arg is an option expecting an argument
+    for opt in "${opts_with_arg[@]}"; do
+        [[ $opt == $prev ]] && return 1
+    done
+
+    # Complete option names
+    COMPREPLY=( $(compgen -W "${opts_without_arg[*]} ${opts_with_arg[*]}" \
+                          -- "$cur") )
+    return 0
+}
+
+complete -F _you-get you-get
diff --git a/contrib/completion/you-get.fish b/contrib/completion/you-get.fish
new file mode 100644
index 0000000000..6917c42241
--- /dev/null
+++ b/contrib/completion/you-get.fish
@@ -0,0 +1,23 @@
+# Fish completion definition for you-get.
+
+complete -c you-get -s V -l version -d 'print version and exit'
+complete -c you-get -s h -l help -d 'print help and exit'
+complete -c you-get -s i -l info -d 'print extracted information'
+complete -c you-get -s u -l url -d 'print extracted information'
+complete -c you-get -l json -d 'print extracted URLs in JSON format'
+complete -c you-get -s n -l no-merge -d 'do not merge video parts'
+complete -c you-get -l no-caption -d 'do not download captions'
+complete -c you-get -s f -l force -d 'force overwrite existing files'
+complete -c you-get -s F -l format -x -d 'set video format to the specified stream id'
+complete -c you-get -s O -l output-filename -d 'set output filename' \
+         -x -a '(__fish_complete_path (commandline -ct) "output filename")'
+complete -c you-get -s o -l output-dir  -d 'set output directory' \
+         -x -a '(__fish_complete_directories (commandline -ct) "output directory")'
+complete -c you-get -s p -l player -x -d 'stream extracted URL to the specified player'
+complete -c you-get -s c -l cookies -d 'load cookies.txt or cookies.sqlite' \
+         -x -a '(__fish_complete_path (commandline -ct) "cookies.txt or cookies.sqlite")'
+complete -c you-get -s x -l http-proxy -x -d 'use the specified HTTP proxy for downloading'
+complete -c you-get -s y -l extractor-proxy -x -d 'use the specified HTTP proxy for extraction only'
+complete -c you-get -l no-proxy -d 'do not use a proxy'
+complete -c you-get -s t -l timeout -x -d 'set socket timeout'
+complete -c you-get -s d -l debug -d 'show traceback and other debug info'

From c372b09e5957b6cfe143d9ce58c56ce224b6128c Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sun, 29 May 2016 17:29:15 -0400
Subject: [PATCH 0151/1225] [Readme] Add pip install from Github

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index 60cb125a75..a972df231b 100644
--- a/README.md
+++ b/README.md
@@ -107,6 +107,12 @@ or download the latest release via:
 $ you-get https://github.com/soimort/you-get/archive/master.zip
 ```
 
+In order to get the latest ```develop``` branch without messing up the PIP, you can try:
+
+```
+$ pip3 install --upgrade git+https://github.com/soimort/you-get@develop
+```
+
 ## Getting Started
 
 ### Download a video

From aaf00414462cf0a7c0d7bb6a131f54e2c16891fc Mon Sep 17 00:00:00 2001
From: zhuzhichao <me@zhuzhichao.com>
Date: Sat, 4 Jun 2016 14:20:42 +0800
Subject: [PATCH 0152/1225] Install via brew (Mac only)

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index a972df231b..d915b0204a 100644
--- a/README.md
+++ b/README.md
@@ -93,6 +93,14 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Option 6: Homebrew (Mac only)
+
+You can install `you-get` easily via:
+
+```
+$ brew install you-get
+```
+
 ## Upgrading
 
 Based on which option you chose to install `you-get`, you may upgrade it via:

From b05fa57a36bb153b847f2c70c00c02ab93f17415 Mon Sep 17 00:00:00 2001
From: Yi Zhi <zhiyi523@gmail.com>
Date: Sun, 5 Jun 2016 10:39:03 +0800
Subject: [PATCH 0153/1225] [Bilibili] download specified P according to the
 link

---
 src/you_get/extractors/bilibili.py | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 55293533fa..d355eabdb6 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -11,12 +11,14 @@
 import hashlib
 import re
 
-appkey='f3bb208b3d081dc8'
+appkey = 'f3bb208b3d081dc8'
+
 
 def get_srt_xml(id):
     url = 'http://comment.bilibili.com/%s.xml' % id
     return get_html(url)
 
+
 def parse_srt_p(p):
     fields = p.split(',')
     assert len(fields) == 8, fields
@@ -44,12 +46,14 @@ def parse_srt_p(p):
 
     return pool, mode, font_size, font_color
 
+
 def parse_srt_xml(xml):
     d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
     for x, y in d:
         p = parse_srt_p(x)
     raise NotImplementedError()
 
+
 def parse_cid_playurl(xml):
     from xml.dom.minidom import parseString
     try:
@@ -59,14 +63,15 @@ def parse_cid_playurl(xml):
     except:
         return []
 
+
 def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
     urls = []
     for cid in cids:
         url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
         urls += [i
-                if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                for i in parse_cid_playurl(get_content(url))]
+                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+                 for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -78,6 +83,7 @@ def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
     url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
     urls = [i
@@ -98,6 +104,7 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
     api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
     urls = parse_cid_playurl(get_content(api_url))
@@ -109,6 +116,7 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
         if not info_only:
             download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
@@ -118,9 +126,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         title = unescape_html(title)
         title = escape_file_path(title)
 
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"', r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
     assert flashvars
-    flashvars = flashvars.replace(': ','=')
+    flashvars = flashvars.replace(': ', '=')
     t, cid = flashvars.split('=', 1)
     cid = cid.split('&')[0]
     if t == 'cid':
@@ -133,7 +142,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
             titles = re.findall('<option value=.*>(.+)</option>', html)
-            for page in pages:
+            for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
                                    r'flashvars="([^"]+)"',
@@ -141,6 +150,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                 if flashvars:
                     t, cid = flashvars.split('=', 1)
                     cids.append(cid.split('&')[0])
+                if url.endswith(page):
+                    cids = [cid.split('&')[0]]
+                    titles = [titles[i]]
+                    break
 
             # no multi-P
             if not pages:
@@ -173,6 +186,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
+
 site_info = "bilibili.com"
 download = bilibili_download
 download_playlist = bilibili_download

From fc83718e5870829dcc2ed1b0db5b01e3df7e277e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 21:23:17 +0200
Subject: [PATCH 0154/1225] [miaopai] fix #1194

---
 src/you_get/extractors/miaopai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 9f195baa08..48a712392d 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -37,7 +37,7 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
         miaopai_download_by_url(url, output_dir, merge, info_only)
     elif re.match(r'http://weibo.com/p/230444\w+', url):
         _fid = match1(url, r'http://weibo.com/p/230444(\w+)')
-        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid))
+        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid), output_dir, merge, info_only)
 
 site_info = "miaopai"
 download = miaopai_download

From 13dc2e9d61dee1db5a7e843a5c1d24c4dca4309d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 22:13:15 +0200
Subject: [PATCH 0155/1225] [facebook] fix #1203

---
 src/you_get/extractors/facebook.py | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 4ed1e2af47..72ce8f317e 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -5,24 +5,21 @@
 from ..common import *
 import json
 
-
 def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
 
-    title = r1(r'<title id="pageTitle">(.+) \| Facebook</title>', html)
-    s2 = parse.unquote(unicodize(r1(r'\["params","([^"]*)"\]', html)))
-    data = json.loads(s2)
-    video_data = data["video_data"]["progressive"]
-    for fmt in ["hd_src", "sd_src"]:
-        src = video_data[0][fmt]
-        if src:
-            break
+    title = r1(r'<title id="pageTitle">(.+)</title>', html)
+    sd_urls = [
+        unicodize(str.replace(i, '\\/', '/'))
+        for i in re.findall(r'"sd_src_no_ratelimit":"([^"]*)"', html)
+    ]
 
-    type, ext, size = url_info(src, True)
+    type, ext, size = url_info(sd_urls[0], True)
+    size = urls_size(sd_urls)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([src], title, ext, size, output_dir, merge=merge)
+        download_urls(sd_urls, title, ext, size, output_dir, merge=False)
 
 site_info = "Facebook.com"
 download = facebook_download

From a256b44e3ecbcbcf41e82cf03977dd22d615cc96 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 23 Jun 2016 22:15:36 +0200
Subject: [PATCH 0156/1225] version 0.4.455

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 5a9d5581bc..61e75ead60 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.424'
+__version__ = '0.4.455'

From 9c8d8b0023d8399e047de0ab503e29f12881118b Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Sat, 25 Jun 2016 15:11:43 +0800
Subject: [PATCH 0157/1225] iqiyi: use html5 to fix #1211 , but lose some
 stream quality

thanks @yan12125

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py    | 49 +++++++---------
 src/you_get/extractors/iqiyi_sc.py | 92 ++++++++++++++++++++++++++++++
 2 files changed, 114 insertions(+), 27 deletions(-)
 create mode 100644 src/you_get/extractors/iqiyi_sc.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 7431b91fdc..320520fa98 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -9,6 +9,10 @@
 from zlib import decompress
 import hashlib
 
+import time
+
+from .iqiyi_sc import gen_sc
+
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -43,6 +47,7 @@
 10 4k
 96 topspeed
 
+'''
 '''
 def mix(tvid):
     salt = '4a1caba4b4465345366f28da7c117d20'
@@ -75,42 +80,32 @@ def getDispathKey(rid):
     time=json.loads(get_content("http://data.video.qiyi.com/t?tn="+str(random())))["t"]
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
+'''
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': '4k', 'container': 'f4v', 'video_profile': '4K'},
-        {'id': 'fullhd', 'container': 'f4v', 'video_profile': '全高清'},
-        {'id': 'suprt-high', 'container': 'f4v', 'video_profile': '超高清'},
-        {'id': 'super', 'container': 'f4v', 'video_profile': '超清'},
-        {'id': 'high', 'container': 'f4v', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'f4v', 'video_profile': '标清'},
-        {'id': 'topspeed', 'container': 'f4v', 'video_profile': '最差'},
+        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
+        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
     ]
 
-    stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
+    supported_stream_types = [ 'high', 'standard']
 
-    stream_urls = {  '4k': [] , 'fullhd' : [], 'suprt-high' : [], 'super' : [], 'high' : [], 'standard' :[], 'topspeed' :[]}
 
-    baseurl = ''
+    stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
 
-    gen_uid = ''
-    def getVMS(self):
+    def getVMS(self,rate):
         #tm ->the flash run time for md5 usage
         #um -> vip 1 normal 0
         #authkey -> for password protected video ,replace '' with your password
         #puid user.passportid may empty?
         #TODO: support password protected video
         tvid, vid = self.vid
-        tm, sc, src = mix(tvid)
-        uid = self.gen_uid
-        vmsreq='http://cache.video.qiyi.com/vms?key=fvip&src=1702633101b340d8917a69cf8a4b8c7' +\
-                "&tvId="+tvid+"&vid="+vid+"&vinfo=1&tm="+tm+\
-                "&enc="+sc+\
-                "&qyid="+uid+"&tn="+str(random()) +"&um=1" +\
-                "&authkey="+hashlib.new('md5',bytes(hashlib.new('md5', b'').hexdigest()+str(tm)+tvid,'utf-8')).hexdigest()
-        return json.loads(get_content(vmsreq))
+        t = int(time.time() * 1000)
+        sc = gen_sc(tvid, t).decode('utf-8')
+        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
+        return json.loads(get_content(vmsreq)[13:])
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -134,13 +129,12 @@ def prepare(self, **kwargs):
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
 
-        self.gen_uid = uuid4().hex
-        try:
-            info = self.getVMS()
-        except:
-            self.download_playlist_by_url(self.url, **kwargs)
-            exit(0)
-
+        for stream in self.supported_stream_types:
+            info = self.getVMS(self.stream_to_bid[stream])
+            if info["code"] == "A00000":
+                self.title = info['data']['playInfo']['vn']
+                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
+'''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
             log.wtf("is your you-get up-to-date?")
@@ -208,6 +202,7 @@ def extract(self, **kwargs):
         #because the url is generated before start downloading
         #and the key may be expired after 10 minutes
         self.streams[stream_id]['src'] = urls
+'''
 
 site = Iqiyi()
 download = site.download_by_url
diff --git a/src/you_get/extractors/iqiyi_sc.py b/src/you_get/extractors/iqiyi_sc.py
new file mode 100644
index 0000000000..4fa4ccdf46
--- /dev/null
+++ b/src/you_get/extractors/iqiyi_sc.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import binascii
+import math
+import time
+
+M = [1732584193, -271733879]
+M.extend([~M[0], ~M[1]])
+I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
+C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
+
+
+def L(n, t):
+    if t is None:
+        t = 0
+    return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
+
+
+def rshift(val, n):
+    return val >> n if val >= 0 else (val+0x100000000) >> n
+
+
+def trunc(n):
+    n = n % 0x100000000
+    if n > 0x7fffffff:
+        n -= 0x100000000
+    return n
+
+
+def gen_sc(tvid, Z):
+    def transform(string, mod):
+        num = int(string, 16)
+        return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
+
+    C = list(C_base)
+    o = list(M)
+    k = str(Z - 7)
+    for i in range(13):
+        a = i
+        C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
+
+    for i in range(16):
+        a = i + 13
+        start = (i >> 2) * 8
+        r = '03967743b643f66763d623d637e30733'
+        C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
+
+    for i in range(16):
+        a = i + 29
+        start = (i >> 2) * 8
+        r = '7038766939776a32776a32706b337139'
+        C[a >> 2] |= transform(r[start:start + 8], 1)
+
+    for i in range(9):
+        a = i + 45
+        if i < len(tvid):
+            C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
+
+    for a in range(64):
+        i = a
+        I = i >> 4
+        C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + rshift(a, 6)
+        m = L(
+                L(
+                    o[0],
+                    [
+                        trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
+                        trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
+                        o[1] ^ o[2] ^ o[3],
+                        o[2] ^ trunc(o[1] | ~o[3])
+                    ][I]
+                ),
+                L(
+                    trunc(int(abs(math.sin(i + 1)) * 4294967296)),
+                    C[C_index] if C_index < len(C) else None
+                )
+            )
+        I = I_table[4 * I + i % 4]
+        o = [
+                o[3],
+                L(o[1], trunc(trunc(m << I) | rshift(m, 32 - I))),
+                o[1],
+                o[2],
+            ]
+
+    new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
+    s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
+    return binascii.hexlify(bytes(s))[1::2]
+
+if __name__ == '__main__':
+    print(gen_sc("494496100", 1466495259194))
+    print(gen_sc("397768800", 1466795077775))
+    print(gen_sc("397768800", 1466796325746))

From d8aca8f5427d4e365da817288906cff9dd88bade Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 26 Jun 2016 16:50:28 +0200
Subject: [PATCH 0158/1225] [bilibili] quick hack for bangumi URLs (fix #1226)

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index d355eabdb6..247825988b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -120,6 +120,11 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        url = r1(r'"([^"]+)" class="v-av-link"', html)
+        html = get_content(url)
+
     title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
                    r'<h1[^>]*>([^<>]+)</h1>'], html)
     if title:

From d370e9952f46d4e56e2abb6a53c8955c7a2f0d7c Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Tue, 28 Jun 2016 16:44:54 +0800
Subject: [PATCH 0159/1225] enable --socks-proxy

---
 src/you_get/common.py | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 100f3869d7..119640d5aa 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1071,12 +1071,13 @@ def version():
     -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
     -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
          --no-proxy                     Never use a proxy.
+    -s | --socks-proxy <HOST:PORT>      Use an SOCKS5 proxy for downloading.
     -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:t:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
     if download_playlist:
         short_opts = 'l' + short_opts
         opts = ['playlist'] + opts
@@ -1104,6 +1105,7 @@ def version():
     lang = None
     output_dir = '.'
     proxy = None
+    socks_proxy = None
     extractor_proxy = None
     traceback = False
     timeout = 600
@@ -1176,6 +1178,8 @@ def version():
             caption = False
         elif o in ('-x', '--http-proxy'):
             proxy = a
+        elif o in ('-s', '--socks-proxy'):
+            socks_proxy = a
         elif o in ('-y', '--extractor-proxy'):
             extractor_proxy = a
         elif o in ('--lang',):
@@ -1189,7 +1193,21 @@ def version():
         print(help)
         sys.exit()
 
-    set_http_proxy(proxy)
+    if (socks_proxy):
+        try:
+          import socket
+          import socks
+          socks_proxy_addrs = socks_proxy.split(':')
+          socks.set_default_proxy(socks.SOCKS5, 
+                                  socks_proxy_addrs[0], 
+                                  int(socks_proxy_addrs[1]))
+          socket.socket = socks.socksocket
+        except ImportError:
+          log.w('Error importing PySocks library, socks proxy ignored.'
+                'In order to use use socks proxy, please install PySocks.')
+    else:
+        import socket
+        set_http_proxy(proxy)
 
     socket.setdefaulttimeout(timeout)
 

From 8bc81d9a70448e5078af86e07c35b12f3f8b1b6a Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Wed, 29 Jun 2016 20:18:18 +0800
Subject: [PATCH 0160/1225] iqiyi: support more stream quality

algorism form @ERioK
thank you @ERioK

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py | 45 +++++++++++++++++++--------------
 1 file changed, 26 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 320520fa98..bda2c2e8a6 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -11,8 +11,6 @@
 
 import time
 
-from .iqiyi_sc import gen_sc
-
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -86,26 +84,31 @@ class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'},
+        {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'},
     ]
-
+    '''
     supported_stream_types = [ 'high', 'standard']
 
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
+    '''
+    ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
+    vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'}
 
-    def getVMS(self,rate):
-        #tm ->the flash run time for md5 usage
-        #um -> vip 1 normal 0
-        #authkey -> for password protected video ,replace '' with your password
-        #puid user.passportid may empty?
-        #TODO: support password protected video
+    def getVMS(self):
         tvid, vid = self.vid
         t = int(time.time() * 1000)
-        sc = gen_sc(tvid, t).decode('utf-8')
-        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
-        return json.loads(get_content(vmsreq)[13:])
+        src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+        sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+        vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+        return json.loads(get_content(vmsreq))
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -128,12 +131,16 @@ def prepare(self, **kwargs):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
+            self.title = match1(html, '<title>([^<]+)').split('-')[0]
+
+        info = self.getVMS()
+        assert info['code'] == 'A00000', 'can\'t play this video'
+
+        for stream in info['data']['vidl']:
+           stream_id = self.vd_2_id[stream['vd']]
+           stream_profile = self.vd_2_profile[stream['vd']]
+           self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
 
-        for stream in self.supported_stream_types:
-            info = self.getVMS(self.stream_to_bid[stream])
-            if info["code"] == "A00000":
-                self.title = info['data']['playInfo']['vn']
-                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")

From 5966a090f478b876b31632f2b654f27c382ebdba Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 15:56:39 -0400
Subject: [PATCH 0161/1225] [Common, ffmpeg]Exp: Add a ffmpeg downloader and
 player for any URL

---
 src/you_get/common.py             | 17 +++++++++
 src/you_get/processor/ffmpeg.py   | 61 +++++++++++++++++++++++++++++++
 src/you_get/processor/rtmpdump.py |  1 +
 3 files changed, 79 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 119640d5aa..c0097a4ed0 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -898,6 +898,23 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re
     assert has_rtmpdump_installed(), "RTMPDump not installed."
     download_rtmpdump_stream(url,  title, ext,params, output_dir)
 
+def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
+    assert url
+    if dry_run:
+        print('Real URL:\n%s\n' % [url])
+        if params.get("-y",False): #None or unset ->False
+            print('Real Playpath:\n%s\n' % [params.get("-y")])
+        return
+
+    if player:
+        from .processor.ffmpeg import ffmpeg_play_stream
+        ffmpeg_play_stream(player, url, params)
+        return
+
+    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_streaming
+    assert has_ffmpeg_installed(), "FFmpeg not installed."
+    ffmpeg_download_stream(url, title, ext, params, output_dir)
+
 def playlist_not_supported(name):
     def f(*args, **kwargs):
         raise NotImplementedError('Playlist is not supported for ' + name)
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index e7ee35d6b7..1d5d850c4d 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -199,3 +199,64 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     for file in files:
         os.remove(file + '.ts')
     return True
+
+def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
+    """str, str->True
+    WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
+    You can basicly download anything with this function
+    but better leave it alone with 
+    """
+    output = title + '.' + ext
+    
+    if not (output_dir == '.'):
+        output = output_dir + output
+        
+    ffmpeg_params = []
+    #should these exist...
+    if len(params) > 0:
+        for k, v in params:
+            ffmpeg_params.append(k)
+            ffmpeg_params.append(v)
+        
+    print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params.append(files)  #not the same here!!!!
+    
+    if FFMPEG == 'avconv':  #who cares?
+        ffmpeg_params += ['-c', 'copy', output]
+    else:
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output]
+    
+    ffmpeg_params.append(output)
+    
+    subprocess.call(ffmpeg_params)
+
+    return True
+
+#
+#To be refactor
+#Direct copy of rtmpdump.py
+#
+def ffmpeg_play_stream(player, url, params={}):
+    ffmpeg_params = []
+    #should these exist...
+    if len(params) > 0:
+        for k, v in params:
+            ffmpeg_params.append(k)
+            ffmpeg_params.append(v)
+        
+    print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params.append(url)  #not the same here!!!!
+    
+    if FFMPEG == 'avconv':  #who cares?
+        ffmpeg_params += ['-c', 'copy', '|']
+    else:
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|']
+    
+    ffmpeg_params += [player, '-']
+    
+    print(' '.join(ffmpeg_params))
+    
+    subprocess.call(ffmpeg_params)
+    return
diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py
index aadb688708..cf5f822cf0 100644
--- a/src/you_get/processor/rtmpdump.py
+++ b/src/you_get/processor/rtmpdump.py
@@ -43,6 +43,7 @@ def download_rtmpdump_stream(url, title, ext,params={},output_dir='.'):
 
 #
 #To be refactor
+#To the future myself: Remember to refactor the same function in ffmpeg.py
 #
 def play_rtmpdump_stream(player, url, params={}):
     cmdline="rtmpdump -r '%s' "%url

From 0d06c260c0ca535d7652e50c60f676f0bdceba14 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 21:28:49 -0400
Subject: [PATCH 0162/1225] [iQiyi]try to fix CI complaint

---
 src/you_get/extractors/iqiyi.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index bda2c2e8a6..4544552d84 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -137,9 +137,9 @@ def prepare(self, **kwargs):
         assert info['code'] == 'A00000', 'can\'t play this video'
 
         for stream in info['data']['vidl']:
-           stream_id = self.vd_2_id[stream['vd']]
-           stream_profile = self.vd_2_profile[stream['vd']]
-           self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            stream_id = self.vd_2_id[stream['vd']]
+            stream_profile = self.vd_2_profile[stream['vd']]
+            self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
 
 '''
         if info["code"] != "A000000":

From 2d542c2cd3b2b6a56886342f53ceb476a08818eb Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 21:54:46 -0400
Subject: [PATCH 0163/1225] [iQiyi]CI complaint CI complaint go away

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 4544552d84..de8756bf05 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -99,7 +99,7 @@ class Iqiyi(VideoExtractor):
     '''
     ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
     vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
-    vd_2_profile = {21: u'超清', 2: u'高清', 4: u'超高清', 17: u'全高清', 96: u'流畅', 1: u'标清'}
+    vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'}
 
     def getVMS(self):
         tvid, vid = self.vid

From fd93b0380d0c04c4766e520e53a6256437e39067 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 23:15:28 -0400
Subject: [PATCH 0164/1225] [FFmpeg] Change arguments to record M3U

---
 src/you_get/processor/ffmpeg.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1d5d850c4d..a387be0a5d 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -219,16 +219,18 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
             ffmpeg_params.append(v)
         
     print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    ffmpeg_params = [FFMPEG] + ['-y', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', output]
     else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', output]
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
     
     ffmpeg_params.append(output)
     
+    print(' '.join(ffmpeg_params))
+    
     subprocess.call(ffmpeg_params)
 
     return True
@@ -252,7 +254,7 @@ def ffmpeg_play_stream(player, url, params={}):
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', '|']
     else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '-bsf:v', 'h264_mp4toannexb', '|']
+        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|']
     
     ffmpeg_params += [player, '-']
     

From 18bc44ea4155d979f8e5fb864e2233d9b3f36eda Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 29 Jun 2016 23:42:19 -0400
Subject: [PATCH 0165/1225] [FFmpeg] Fix stream corrupted if Ctrl+C

Use q instead.
---
 src/you_get/processor/ffmpeg.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index a387be0a5d..c23b0eaf66 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -218,8 +218,8 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
             ffmpeg_params.append(k)
             ffmpeg_params.append(v)
         
-    print('Downloading streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + ['-y', '-i']
+    print('Downloading streaming content with FFmpeg, press q to stop recording...')
+    ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
@@ -231,7 +231,14 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     
     print(' '.join(ffmpeg_params))
     
-    subprocess.call(ffmpeg_params)
+    try:
+        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
+        a.communicate()
+    except KeyboardInterrupt:
+        try:
+            a.stdin.write('q'.encode('utf-8'))
+        except:
+            pass
 
     return True
 
@@ -247,8 +254,8 @@ def ffmpeg_play_stream(player, url, params={}):
             ffmpeg_params.append(k)
             ffmpeg_params.append(v)
         
-    print('Playing streaming content with FFmpeg, press Ctrl+C to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
+    print('Playing streaming content with FFmpeg, press 1 to stop recording...')
+    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']
     ffmpeg_params.append(url)  #not the same here!!!!
     
     if FFMPEG == 'avconv':  #who cares?
@@ -260,5 +267,13 @@ def ffmpeg_play_stream(player, url, params={}):
     
     print(' '.join(ffmpeg_params))
     
-    subprocess.call(ffmpeg_params)
-    return
+    try:
+        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
+        a.communicate()
+    except KeyboardInterrupt:
+        try:
+            a.stdin.write('q'.encode('utf-8'))
+        except:
+            pass
+
+    return True
\ No newline at end of file

From 62a9e35ed0cc0aebb9ef13edb76394c9efd543e6 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 30 Jun 2016 02:36:07 -0400
Subject: [PATCH 0166/1225] [CNTV] Fix CCAV domain, Add pattern, fix #1193
 ,#909 , replace #1031

---
 src/you_get/common.py          | 1 +
 src/you_get/extractors/cntv.py | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 119640d5aa..d30397c757 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -9,6 +9,7 @@
     'bandcamp'         : 'bandcamp',
     'baomihua'         : 'baomihua',
     'bilibili'         : 'bilibili',
+    'cctv'             : 'cntv',
     'cntv'             : 'cntv',
     'cbs'              : 'cbs',
     'dailymotion'      : 'dailymotion',
diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index cfd96e59b7..a32808cb54 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -7,6 +7,7 @@
 import json
 import re
 
+
 def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
     assert id
     info = json.loads(get_html('http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=' + id))
@@ -31,7 +32,12 @@ def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
-    elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url):
+    elif re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url):
+        html = get_content(url)
+        id = match1(html, r'guid = \"(.+)\"')
+    elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
+         re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
+         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url): 
         id = r1(r'videoCenterId","(\w+)"', get_html(url))
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
         id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)

From 48159051de49b16b4d3b721baa076c9416dbfe75 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 30 Jun 2016 02:43:41 -0400
Subject: [PATCH 0167/1225] [CNTV] reduce code

---
 src/you_get/extractors/cntv.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index a32808cb54..e25fa96116 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -32,12 +32,11 @@ def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
-    elif re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url):
-        html = get_content(url)
-        id = match1(html, r'guid = \"(.+)\"')
     elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
-         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url): 
+         re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \
+         re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
+         re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url): 
         id = r1(r'videoCenterId","(\w+)"', get_html(url))
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
         id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)

From 78ee9fffe32f4738264554b854c99a572b7fa4f4 Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Wed, 29 Jun 2016 20:18:18 +0800
Subject: [PATCH 0168/1225] iqiyi: support more stream quality

algorism form @ERioK
thank you @ERioK

v2. remove iqiyi_sc.py, and remove Unicode literal syntax
v3. indent mistake
v4. support vip streams
v5. combine stream profile by size
v6. support 4k profiles

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py    | 81 +++++++++++++++++++-------
 src/you_get/extractors/iqiyi_sc.py | 92 ------------------------------
 2 files changed, 60 insertions(+), 113 deletions(-)
 delete mode 100644 src/you_get/extractors/iqiyi_sc.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 320520fa98..9761f3d1e1 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -8,11 +8,10 @@
 from math import floor
 from zlib import decompress
 import hashlib
+from ..util import log
 
 import time
 
-from .iqiyi_sc import gen_sc
-
 '''
 Changelog:
 -> http://www.iqiyi.com/common/flashplayer/20150916/MainPlayer_5_2_28_c3_3_7_4.swf
@@ -81,31 +80,36 @@ def getDispathKey(rid):
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
 '''
+def getVMS(tvid, vid):
+    t = int(time.time() * 1000)
+    src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+    key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+    sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+    vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+    return json.loads(get_content(vmsreq))
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'high', 'container': 'mp4', 'video_profile': '高清'},
-        {'id': 'standard', 'container': 'mp4', 'video_profile': '标清'},
+        {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'},
     ]
-
+    '''
     supported_stream_types = [ 'high', 'standard']
 
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
+    '''
+    ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'}
+    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'}
+
 
-    def getVMS(self,rate):
-        #tm ->the flash run time for md5 usage
-        #um -> vip 1 normal 0
-        #authkey -> for password protected video ,replace '' with your password
-        #puid user.passportid may empty?
-        #TODO: support password protected video
-        tvid, vid = self.vid
-        t = int(time.time() * 1000)
-        sc = gen_sc(tvid, t).decode('utf-8')
-        vmsreq= 'http://cache.m.iqiyi.com/jp/tmts/{}/{}/?platForm=h5&rate={}&tvid={}&vid={}&cupid=qc_100001_100186&type=mp4&olimit=0&agenttype=13&src=d846d0c32d664d32b6b54ea48997a589&sc={}&t={}&__jsT=null'.format(tvid, vid, rate, tvid, vid, sc, t - 7)
-        return json.loads(get_content(vmsreq)[13:])
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -128,12 +132,47 @@ def prepare(self, **kwargs):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
+            self.title = match1(html, '<title>([^<]+)').split('-')[0]
+        tvid, videoid = self.vid
+        info = getVMS(tvid, videoid)
+        assert info['code'] == 'A00000', 'can\'t play this video'
+
+        for stream in info['data']['vidl']:
+            try:
+                stream_id = self.vd_2_id[stream['vd']]
+                if stream_id in self.stream_types:
+                    continue
+                stream_profile = self.id_2_profile[stream_id]
+                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            except:
+                log.i("vd: {} is not handled".format(stream['vd']))
+                log.i("info is {}".format(stream))
+        # why I need do below???
+        if not 'BD' in self.stream_types:
+            p1080_vids = []
+            if 18 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
+            if 5 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            for v in p1080_vids:
+                p1080_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    p1080_url = p1080_info['data']['m3u']
+                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
+                    break
 
-        for stream in self.supported_stream_types:
-            info = self.getVMS(self.stream_to_bid[stream])
-            if info["code"] == "A00000":
-                self.title = info['data']['playInfo']['vn']
-                self.streams[stream] = {'container': 'mp4', 'video_profile': stream, 'src' : [info['data']['m3u']], 'size' : url_size(info['data']['m3u'])}
+        if not '4k' in self.stream_types:
+            k4_vids = []
+            if 19 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
+            if 10 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            for v in k4_vids:
+                k4_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    k4_url = k4_info['data']['m3u']
+                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
+                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
diff --git a/src/you_get/extractors/iqiyi_sc.py b/src/you_get/extractors/iqiyi_sc.py
deleted file mode 100644
index 4fa4ccdf46..0000000000
--- a/src/you_get/extractors/iqiyi_sc.py
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env python
-import binascii
-import math
-import time
-
-M = [1732584193, -271733879]
-M.extend([~M[0], ~M[1]])
-I_table = [7, 12, 17, 22, 5, 9, 14, 20, 4, 11, 16, 23, 6, 10, 15, 21]
-C_base = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8388608, 432]
-
-
-def L(n, t):
-    if t is None:
-        t = 0
-    return trunc(((n >> 1) + (t >> 1) << 1) + (n & 1) + (t & 1))
-
-
-def rshift(val, n):
-    return val >> n if val >= 0 else (val+0x100000000) >> n
-
-
-def trunc(n):
-    n = n % 0x100000000
-    if n > 0x7fffffff:
-        n -= 0x100000000
-    return n
-
-
-def gen_sc(tvid, Z):
-    def transform(string, mod):
-        num = int(string, 16)
-        return (num >> 8 * (i % 4) & 255 ^ i % mod) << ((a & 3) << 3)
-
-    C = list(C_base)
-    o = list(M)
-    k = str(Z - 7)
-    for i in range(13):
-        a = i
-        C[a >> 2] |= ord(k[a]) << 8 * (a % 4)
-
-    for i in range(16):
-        a = i + 13
-        start = (i >> 2) * 8
-        r = '03967743b643f66763d623d637e30733'
-        C[a >> 2] |= transform(''.join(reversed(r[start:start + 8])), 7)
-
-    for i in range(16):
-        a = i + 29
-        start = (i >> 2) * 8
-        r = '7038766939776a32776a32706b337139'
-        C[a >> 2] |= transform(r[start:start + 8], 1)
-
-    for i in range(9):
-        a = i + 45
-        if i < len(tvid):
-            C[a >> 2] |= ord(tvid[i]) << 8 * (a % 4)
-
-    for a in range(64):
-        i = a
-        I = i >> 4
-        C_index = [i, 5 * i + 1, 3 * i + 5, 7 * i][I] % 16 + rshift(a, 6)
-        m = L(
-                L(
-                    o[0],
-                    [
-                        trunc(o[1] & o[2]) | trunc(~o[1] & o[3]),
-                        trunc(o[3] & o[1]) | trunc(~o[3] & o[2]),
-                        o[1] ^ o[2] ^ o[3],
-                        o[2] ^ trunc(o[1] | ~o[3])
-                    ][I]
-                ),
-                L(
-                    trunc(int(abs(math.sin(i + 1)) * 4294967296)),
-                    C[C_index] if C_index < len(C) else None
-                )
-            )
-        I = I_table[4 * I + i % 4]
-        o = [
-                o[3],
-                L(o[1], trunc(trunc(m << I) | rshift(m, 32 - I))),
-                o[1],
-                o[2],
-            ]
-
-    new_M = [L(o[0], M[0]), L(o[1], M[1]), L(o[2], M[2]), L(o[3], M[3])]
-    s = [new_M[a >> 3] >> (1 ^ a & 7) * 4 & 15 for a in range(32)]
-    return binascii.hexlify(bytes(s))[1::2]
-
-if __name__ == '__main__':
-    print(gen_sc("494496100", 1466495259194))
-    print(gen_sc("397768800", 1466795077775))
-    print(gen_sc("397768800", 1466796325746))

From 7452a4bb0e7aac32db89d871e7636741fddf641a Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 1 Jul 2016 00:38:26 -0400
Subject: [PATCH 0169/1225] [iQiyi] fix for PR conflict

---
 src/you_get/extractors/iqiyi.py | 80 +++++++++++++++++++++++----------
 1 file changed, 56 insertions(+), 24 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/iqiyi.py

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
old mode 100644
new mode 100755
index de8756bf05..a1552fe456
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -8,6 +8,7 @@
 from math import floor
 from zlib import decompress
 import hashlib
+from ..util import log
 
 import time
 
@@ -79,17 +80,24 @@ def getDispathKey(rid):
     t=str(int(floor(int(time)/(10*60.0))))
     return hashlib.new("md5",bytes(t+tp+rid,"utf-8")).hexdigest()
 '''
+def getVMS(tvid, vid):
+    t = int(time.time() * 1000)
+    src = '76f90cbd92f94a2e925d83e8ccd22cb7'
+    key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
+    sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
+    vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
+    return json.loads(get_content(vmsreq))
 
 class Iqiyi(VideoExtractor):
     name = "爱奇艺 (Iqiyi)"
 
     stream_types = [
-        {'id': 'BD', 'container': 'm3u8', 'video_profile': '全高清'},
-        {'id': 'FD', 'container': 'm3u8', 'video_profile': '超高清'},
-        {'id': 'TD', 'container': 'm3u8', 'video_profile': '超清'},
-        {'id': 'HD', 'container': 'm3u8', 'video_profile': '高清'},
-        {'id': 'SD', 'container': 'm3u8', 'video_profile': '标清'},
-        {'id': 'LD', 'container': 'm3u8', 'video_profile': '流畅'},
+        {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'},
+        {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'},
+        {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'},
+        {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'},
+        {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'},
+        {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'},
     ]
     '''
     supported_stream_types = [ 'high', 'standard']
@@ -97,18 +105,11 @@ class Iqiyi(VideoExtractor):
 
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
     '''
-    ids = ['BD', 'FD', 'OD', 'TD', 'HD', 'SD', 'LD']
-    vd_2_id = {21: 'TD', 2: 'HD', 4: 'FD', 17: 'BD', 96: 'LD', 1: 'SD'}
-    vd_2_profile = {21: '超清', 2: '高清', 4: '超高清', 17: '全高清', 96: '流畅', 1: '标清'}
-
-    def getVMS(self):
-        tvid, vid = self.vid
-        t = int(time.time() * 1000)
-        src = '76f90cbd92f94a2e925d83e8ccd22cb7'
-        key = 'd5fb4bd9d50c4be6948c97edd7254b0e'
-        sc = hashlib.new('md5', bytes(str(t) + key  + vid, 'utf-8')).hexdigest()
-        vmsreq= url = 'http://cache.m.iqiyi.com/tmts/{0}/{1}/?t={2}&sc={3}&src={4}'.format(tvid,vid,t,sc,src)
-        return json.loads(get_content(vmsreq))
+    ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD']
+    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'}
+    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'}
+
+
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -132,15 +133,46 @@ def prepare(self, **kwargs):
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
             self.title = match1(html, '<title>([^<]+)').split('-')[0]
-
-        info = self.getVMS()
+        tvid, videoid = self.vid
+        info = getVMS(tvid, videoid)
         assert info['code'] == 'A00000', 'can\'t play this video'
 
         for stream in info['data']['vidl']:
-            stream_id = self.vd_2_id[stream['vd']]
-            stream_profile = self.vd_2_profile[stream['vd']]
-            self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            try:
+                stream_id = self.vd_2_id[stream['vd']]
+                if stream_id in self.stream_types:
+                    continue
+                stream_profile = self.id_2_profile[stream_id]
+                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
+            except:
+                log.i("vd: {} is not handled".format(stream['vd']))
+                log.i("info is {}".format(stream))
+        # why I need do below???
+        if not 'BD' in self.stream_types:
+            p1080_vids = []
+            if 18 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
+            if 5 in info['data']['ctl']['vip']['bids']:
+                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            for v in p1080_vids:
+                p1080_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    p1080_url = p1080_info['data']['m3u']
+                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
+                    break
 
+        if not '4k' in self.stream_types:
+            k4_vids = []
+            if 19 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
+            if 10 in info['data']['ctl']['vip']['bids']:
+                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            for v in k4_vids:
+                k4_info = getVMS(tvid, v)
+                if info['code'] == 'A00000':
+                    k4_url = k4_info['data']['m3u']
+                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
+                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")
@@ -214,4 +246,4 @@ def extract(self, **kwargs):
 site = Iqiyi()
 download = site.download_by_url
 iqiyi_download_by_vid = site.download_by_vid
-download_playlist = site.download_playlist_by_url
+download_playlist = site.download_playlist_by_url
\ No newline at end of file

From 5a99b2f42f1157c6b51be768d69fb771b8761405 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 1 Jul 2016 01:07:32 -0400
Subject: [PATCH 0170/1225] [Showroom] Add support via HLS M3U

---
 src/you_get/common.py              |  3 +-
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/showroom.py | 67 ++++++++++++++++++++++++++++++
 3 files changed, 70 insertions(+), 1 deletion(-)
 create mode 100755 src/you_get/extractors/showroom.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 99ce14c7df..c8843ddf13 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -62,6 +62,7 @@
     'pptv'             : 'pptv',
     'qianmo'           : 'qianmo',
     'qq'               : 'qq',
+    'showroom-live'    : 'showroom',
     'sina'             : 'sina',
     'smgbb'            : 'bilibili',
     'sohu'             : 'sohu',
@@ -912,7 +913,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
         ffmpeg_play_stream(player, url, params)
         return
 
-    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_streaming
+    from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
     assert has_ffmpeg_installed(), "FFmpeg not installed."
     ffmpeg_download_stream(url, title, ext, params, output_dir)
 
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 20a7f7cf0e..b879b83e36 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -55,6 +55,7 @@
 from .qianmo import *
 from .qie import *
 from .qq import *
+from .showroom import *
 from .sina import *
 from .sohu import *
 from .soundcloud import *
diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
new file mode 100755
index 0000000000..43d7b19286
--- /dev/null
+++ b/src/you_get/extractors/showroom.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+
+__all__ = ['showroom_download']
+
+from ..common import *
+import urllib.error
+from json import loads
+from time import time
+
+#----------------------------------------------------------------------
+def showroom_get_roomid_by_room_url_key(room_url_key):
+    """str->str"""
+    fake_headers_mobile = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
+    }
+    webpage_url = 'https://www.showroom-live.com/' + room_url_key
+    html = get_content(webpage_url, headers = fake_headers_mobile)
+    roomid = match1(html, r'room\?room_id\=(\d+)')
+    assert roomid
+    return roomid
+
+def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_only = False, **kwargs):
+    '''Source: Android mobile'''
+    timestamp = str(int(time() * 1000))
+    api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
+    html = get_content(api_endpoint)
+    html = json.loads(html)
+    #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
+    if len(html) < 1:
+        log.wtf('Cannot find any live URL! Maybe the live have ended or haven\'t start yet?')
+        
+    #This is mainly for testing the M3U FFmpeg parser so I would ignore any non-m3u ones
+    stream_url = [i['url'] for i in html['streaming_url_list'] if i['is_default'] and i['type'] == 'hls'][0]
+    
+    assert stream_url
+    
+    #title
+    title = ''
+    profile_api = 'https://www.showroom-live.com/api/room/profile?room_id={room_id}'.format(room_id = room_id)
+    html = loads(get_content(profile_api))
+    try:
+        title = html['main_name']
+    except KeyError:
+        title = 'Showroom_{room_id}'.format(room_id = room_id)
+    
+    type_, ext, size = url_info(stream_url)
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_url_ffmpeg(url=stream_url, title=title, ext= 'mp4', output_dir=output_dir)
+    
+
+#----------------------------------------------------------------------
+def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    """"""
+    if re.match( r'(\w+)://www.showroom-live.com/(\w*)', url):
+        room_url_key = match1(url, r'\w+://www.showroom-live.com/(\w*)')
+        room_id = showroom_get_roomid_by_room_url_key(room_url_key)
+        showroom_download_by_room_id(room_id, output_dir, merge, 
+                                    info_only)
+
+site_info = "Showroom"
+download = showroom_download
+download_playlist = playlist_not_supported('showroom')
\ No newline at end of file

From bbe1376a8670060f51803ed11d0027a3ab9792da Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 15:35:02 +0200
Subject: [PATCH 0171/1225] [showroom] support room URLs with dashes (-)

---
 src/you_get/extractors/showroom.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
index 43d7b19286..d0f636bc01 100755
--- a/src/you_get/extractors/showroom.py
+++ b/src/you_get/extractors/showroom.py
@@ -56,8 +56,8 @@ def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_
 #----------------------------------------------------------------------
 def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     """"""
-    if re.match( r'(\w+)://www.showroom-live.com/(\w*)', url):
-        room_url_key = match1(url, r'\w+://www.showroom-live.com/(\w*)')
+    if re.match( r'(\w+)://www.showroom-live.com/([-\w]+)', url):
+        room_url_key = match1(url, r'\w+://www.showroom-live.com/([-\w]+)')
         room_id = showroom_get_roomid_by_room_url_key(room_url_key)
         showroom_download_by_room_id(room_id, output_dir, merge, 
                                     info_only)

From 7516028dd860d642bb1620fef0d78ed6a06153d8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 15:58:57 +0200
Subject: [PATCH 0172/1225] =?UTF-8?q?extractors:=20revert=20permissions=20?=
 =?UTF-8?q?(100755=20=E2=86=92=20100644)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/__init__.py | 0
 src/you_get/extractors/acfun.py    | 0
 src/you_get/extractors/baidu.py    | 0
 src/you_get/extractors/baomihua.py | 0
 src/you_get/extractors/dilidili.py | 0
 src/you_get/extractors/funshion.py | 0
 src/you_get/extractors/iqiyi.py    | 0
 src/you_get/extractors/khan.py     | 0
 src/you_get/extractors/miomio.py   | 0
 src/you_get/extractors/showroom.py | 0
 src/you_get/extractors/yixia.py    | 0
 11 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/extractors/__init__.py
 mode change 100755 => 100644 src/you_get/extractors/acfun.py
 mode change 100755 => 100644 src/you_get/extractors/baidu.py
 mode change 100755 => 100644 src/you_get/extractors/baomihua.py
 mode change 100755 => 100644 src/you_get/extractors/dilidili.py
 mode change 100755 => 100644 src/you_get/extractors/funshion.py
 mode change 100755 => 100644 src/you_get/extractors/iqiyi.py
 mode change 100755 => 100644 src/you_get/extractors/khan.py
 mode change 100755 => 100644 src/you_get/extractors/miomio.py
 mode change 100755 => 100644 src/you_get/extractors/showroom.py
 mode change 100755 => 100644 src/you_get/extractors/yixia.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/khan.py b/src/you_get/extractors/khan.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/miomio.py b/src/you_get/extractors/miomio.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
old mode 100755
new mode 100644
diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
old mode 100755
new mode 100644

From e84810c4d2c1242bbe14265919f8627b9c1e7e54 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 16:42:04 +0200
Subject: [PATCH 0173/1225] [facebook] use hd_src whenever possible and remove
 duplicates

---
 src/you_get/extractors/facebook.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 72ce8f317e..2a96fcb03b 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -9,17 +9,22 @@ def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     html = get_html(url)
 
     title = r1(r'<title id="pageTitle">(.+)</title>', html)
-    sd_urls = [
+    sd_urls = list(set([
         unicodize(str.replace(i, '\\/', '/'))
         for i in re.findall(r'"sd_src_no_ratelimit":"([^"]*)"', html)
-    ]
+    ]))
+    hd_urls = list(set([
+        unicodize(str.replace(i, '\\/', '/'))
+        for i in re.findall(r'"hd_src_no_ratelimit":"([^"]*)"', html)
+    ]))
+    urls = hd_urls if hd_urls else sd_urls
 
-    type, ext, size = url_info(sd_urls[0], True)
-    size = urls_size(sd_urls)
+    type, ext, size = url_info(urls[0], True)
+    size = urls_size(urls)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls(sd_urls, title, ext, size, output_dir, merge=False)
+        download_urls(urls, title, ext, size, output_dir, merge=False)
 
 site_info = "Facebook.com"
 download = facebook_download

From 17386968c7315347ef7384f1a895afbf44d0b6fe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2016 20:28:03 +0200
Subject: [PATCH 0174/1225] [tumblr] fix #1232

---
 src/you_get/extractors/tumblr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index fea061cecd..0c59f25a01 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -68,7 +68,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         real_url = r1(r'<source src="([^"]*)"', html)
     if not real_url:
         iframe_url = r1(r'<[^>]+tumblr_video_container[^>]+><iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
-        if len(iframe_url) > 0:
+        if iframe_url:
             iframe_html = get_content(iframe_url, headers=fake_headers)
             real_url = r1(r'<video[^>]*>[\n ]*<source[^>]+src=[\'"]([^\'"]*)[\'"]', iframe_html)
         else:

From a4ed7859661e8bc9d7c9eab1a0c04eaf8fc7836d Mon Sep 17 00:00:00 2001
From: Zhang Ning <zhangn1985@gmail.com>
Date: Sat, 2 Jul 2016 06:54:35 +0800
Subject: [PATCH 0175/1225] iqiyi: fix when no vip info

Signed-off-by: Zhang Ning <zhangn1985@gmail.com>
---
 src/you_get/extractors/iqiyi.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index a1552fe456..dc9b05fd82 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -148,12 +148,17 @@ def prepare(self, **kwargs):
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
         # why I need do below???
-        if not 'BD' in self.stream_types:
+        try:
+            vip_vds = info['data']['ctl']['vip']['bids']
+            vip_conf = info['data']['ctl']['configs']
+        except:
+            return
+        if not 'BD' in self.streams.keys():
             p1080_vids = []
-            if 18 in info['data']['ctl']['vip']['bids']:
-                p1080_vids.append(info['data']['ctl']['configs']['18']['vid'])
-            if 5 in info['data']['ctl']['vip']['bids']:
-                p1080_vids.append(info['data']['ctl']['configs']['5']['vid'])
+            if 18 in vip_vds:
+                p1080_vids.append(vip_conf['18']['vid'])
+            if 5 in vip_vds:
+                p1080_vids.append(vip_conf['5']['vid'])
             for v in p1080_vids:
                 p1080_info = getVMS(tvid, v)
                 if info['code'] == 'A00000':
@@ -161,12 +166,12 @@ def prepare(self, **kwargs):
                     self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
                     break
 
-        if not '4k' in self.stream_types:
+        if not '4k' in self.streams.keys():
             k4_vids = []
-            if 19 in info['data']['ctl']['vip']['bids']:
-                k4_vids.append(info['data']['ctl']['configs']['19']['vid'])
-            if 10 in info['data']['ctl']['vip']['bids']:
-                k4_vids.append(info['data']['ctl']['configs']['10']['vid'])
+            if 19 in vip_vds:
+                k4_vids.append(vip_conf['19']['vid'])
+            if 10 in vip_vds:
+                k4_vids.append(vip_conf['10']['vid'])
             for v in k4_vids:
                 k4_info = getVMS(tvid, v)
                 if info['code'] == 'A00000':
@@ -246,4 +251,4 @@ def extract(self, **kwargs):
 site = Iqiyi()
 download = site.download_by_url
 iqiyi_download_by_vid = site.download_by_vid
-download_playlist = site.download_playlist_by_url
\ No newline at end of file
+download_playlist = site.download_playlist_by_url

From 3b2b82c2cb6e56cc59228818cf8c2879bdf2e6a9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Jul 2016 14:04:04 +0200
Subject: [PATCH 0176/1225] [iqiyi] remove invalid VIP profiles

See: https://github.com/soimort/you-get/commit/7452a4bb0e7aac32db89d871e7636741fddf641a#commitcomment-18107708
---
 src/you_get/extractors/iqiyi.py | 30 ------------------------------
 1 file changed, 30 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index dc9b05fd82..e9ee5afb94 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -147,37 +147,7 @@ def prepare(self, **kwargs):
             except:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
-        # why I need do below???
-        try:
-            vip_vds = info['data']['ctl']['vip']['bids']
-            vip_conf = info['data']['ctl']['configs']
-        except:
-            return
-        if not 'BD' in self.streams.keys():
-            p1080_vids = []
-            if 18 in vip_vds:
-                p1080_vids.append(vip_conf['18']['vid'])
-            if 5 in vip_vds:
-                p1080_vids.append(vip_conf['5']['vid'])
-            for v in p1080_vids:
-                p1080_info = getVMS(tvid, v)
-                if info['code'] == 'A00000':
-                    p1080_url = p1080_info['data']['m3u']
-                    self.streams['BD'] = {'video_profile': '1080p', 'container': 'm3u8', 'src': [p1080_url], 'size' : 0}
-                    break
 
-        if not '4k' in self.streams.keys():
-            k4_vids = []
-            if 19 in vip_vds:
-                k4_vids.append(vip_conf['19']['vid'])
-            if 10 in vip_vds:
-                k4_vids.append(vip_conf['10']['vid'])
-            for v in k4_vids:
-                k4_info = getVMS(tvid, v)
-                if info['code'] == 'A00000':
-                    k4_url = k4_info['data']['m3u']
-                    self.streams['4k'] = {'video_profile': '4k', 'container': 'm3u8', 'src': [k4_url], 'size' : 0}
-                    break
 '''
         if info["code"] != "A000000":
             log.e("[error] outdated iQIYI key")

From 04fe6cd5a406bb0767314d3a66cdb75b479c050d Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Thu, 7 Jul 2016 16:57:38 -0400
Subject: [PATCH 0177/1225] you get vk photos

---
 src/you_get/extractors/vk.py | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index c83dc48ec2..6ad6624d8c 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -4,7 +4,8 @@
 
 from ..common import *
 
-def vk_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+
+def get_video_info(url):
     video_page = get_content(url)
     title = unescape_html(r1(r'"title":"([^"]+)"', video_page))
     info = dict(re.findall(r'\\"url(\d+)\\":\\"([^"]+)\\"', video_page))
@@ -13,12 +14,39 @@ def vk_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             url = re.sub(r'\\\\\\/', r'/', info[quality])
             break
     assert url
-
     type, ext, size = url_info(url)
+    print_info(site_info, title, type, size)
+
+    return url, title, ext, size
+
+
+def get_image_info(url):
+    image_page = get_content(url)
+    # used for title - vk page owner
+    page_of = re.findall(r'Sender:</dt><dd><a href=.*>(.[^>]+?)</a', image_page)
+    # used for title - date when photo was uploaded
+    photo_date = re.findall(r'<span class="item_date">(.[^>]+?)</span', image_page)
 
+    title = (' ').join(page_of + photo_date)
+    image_link = r1(r'href="([^"]+)" class=\"mva_item\" target="_blank">Download full size', image_page)
+    type, ext, size = url_info(image_link)
     print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge=merge)
+
+    return image_link, title, ext, size
+
+
+def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
+    link = None
+    if re.match(r'vk.com/photo', url):
+        link, title, ext, size = get_video_info(url)
+    elif re.match(r'(.+)vk\.com\/photo(.+)', url):
+        link, title, ext, size = get_image_info(url)
+    else:
+        raise NotImplementedError('Nothing to download here')
+
+    if not info_only and link is not None:
+        download_urls([link], title, ext, size, output_dir, merge=merge)
+
 
 site_info = "VK.com"
 download = vk_download

From 3aa73fc1816e6f76b007599730f2ec5315896370 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 7 Jul 2016 20:09:56 -0400
Subject: [PATCH 0178/1225] [iQiyi] Use FFmpeg to record M3U file

---
 src/you_get/extractors/iqiyi.py | 65 +++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index e9ee5afb94..248446a779 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -147,6 +147,71 @@ def prepare(self, **kwargs):
             except:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
+    
+
+    def download(self, **kwargs):
+        """Override the original one
+        Ugly ugly dirty hack"""
+        if 'json_output' in kwargs and kwargs['json_output']:
+            json_output.output(self)
+        elif 'info_only' in kwargs and kwargs['info_only']:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Display the stream
+                stream_id = kwargs['stream_id']
+                if 'index' not in kwargs:
+                    self.p(stream_id)
+                else:
+                    self.p_i(stream_id)
+            else:
+                # Display all available streams
+                if 'index' not in kwargs:
+                    self.p([])
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    self.p_i(stream_id)
+
+        else:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Download the stream
+                stream_id = kwargs['stream_id']
+            else:
+                # Download stream with the best quality
+                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+
+            if 'index' not in kwargs:
+                self.p(stream_id)
+            else:
+                self.p_i(stream_id)
+
+            if stream_id in self.streams:
+                urls = self.streams[stream_id]['src']
+                ext = self.streams[stream_id]['container']
+                total_size = self.streams[stream_id]['size']
+            else:
+                urls = self.dash_streams[stream_id]['src']
+                ext = self.dash_streams[stream_id]['container']
+                total_size = self.dash_streams[stream_id]['size']
+
+            if not urls:
+                log.wtf('[Failed] Cannot extract video source.')
+            # For legacy main()
+            
+            #Here's the change!!
+            download_url_ffmpeg(urls[0], self.title, 'mp4',
+                          output_dir=kwargs['output_dir'],
+                          merge=kwargs['merge'],)
+
+            if not kwargs['caption']:
+                print('Skipping captions.')
+                return
+            for lang in self.caption_tracks:
+                filename = '%s.%s.srt' % (get_filename(self.title), lang)
+                print('Saving %s ... ' % filename, end="", flush=True)
+                srt = self.caption_tracks[lang]
+                with open(os.path.join(kwargs['output_dir'], filename),
+                          'w', encoding='utf-8') as x:
+                    x.write(srt)
+                print('Done.')    
 
 '''
         if info["code"] != "A000000":

From 13b63aa7733f713da01b63c958e204744ee71328 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Jul 2016 19:12:03 +0200
Subject: [PATCH 0179/1225] version 0.4.486

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 61e75ead60..46850c8a22 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.455'
+__version__ = '0.4.486'

From 50c911507b6d32522d2856cff6cf4460bd9893f6 Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Fri, 8 Jul 2016 15:12:21 -0400
Subject: [PATCH 0180/1225] Lets fix vk videos

---
 README.md                    |  2 +-
 src/you_get/extractors/vk.py | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index aa7f61a090..d3d44b334b 100644
--- a/README.md
+++ b/README.md
@@ -318,7 +318,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | :--: | :-- | :-----: | :-----: | :-----: |
 | **YouTube** | <https://www.youtube.com/>    |✓| | |
 | **Twitter** | <https://twitter.com/>        |✓|✓| |
-| VK          | <http://vk.com/>              |✓| | |
+| VK          | <http://vk.com/>              |✓|✓| |
 | Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
 | Vidto       | <http://vidto.me/>            |✓| | |
diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index 6ad6624d8c..ea3e3851bc 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -7,12 +7,14 @@
 
 def get_video_info(url):
     video_page = get_content(url)
-    title = unescape_html(r1(r'"title":"([^"]+)"', video_page))
-    info = dict(re.findall(r'\\"url(\d+)\\":\\"([^"]+)\\"', video_page))
+    title = r1(r'<div class="vv_summary">(.[^>]+?)</div', video_page)
+    sources = re.findall(r'<source src=\"(.[^>]+?)"', video_page)
+
     for quality in ['1080', '720', '480', '360', '240']:
-        if quality in info:
-            url = re.sub(r'\\\\\\/', r'/', info[quality])
-            break
+        for source in sources:
+            if source.find(quality) != -1:
+                url = source
+                break
     assert url
     type, ext, size = url_info(url)
     print_info(site_info, title, type, size)
@@ -37,7 +39,7 @@ def get_image_info(url):
 
 def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
     link = None
-    if re.match(r'vk.com/photo', url):
+    if re.match(r'(.+)z\=video(.+)', url):
         link, title, ext, size = get_video_info(url)
     elif re.match(r'(.+)vk\.com\/photo(.+)', url):
         link, title, ext, size = get_image_info(url)

From 4793bc30d03bf104458cadadbed30ed30f96901a Mon Sep 17 00:00:00 2001
From: misha shelemetyev <misha.shelemetyev@onforce.com>
Date: Fri, 8 Jul 2016 15:35:07 -0400
Subject: [PATCH 0181/1225] 1248 surround quality by dots to avoid issues with
 having q value in uri

---
 src/you_get/extractors/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index ea3e3851bc..98f3471b85 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -10,7 +10,7 @@ def get_video_info(url):
     title = r1(r'<div class="vv_summary">(.[^>]+?)</div', video_page)
     sources = re.findall(r'<source src=\"(.[^>]+?)"', video_page)
 
-    for quality in ['1080', '720', '480', '360', '240']:
+    for quality in ['.1080.', '.720.', '.480.', '.360.', '.240.']:
         for source in sources:
             if source.find(quality) != -1:
                 url = source

From 01cc77c96acf6f961111732d345d0ad23eaa3d40 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 10 Jul 2016 07:50:51 +0200
Subject: [PATCH 0182/1225] [showroom] do polling on offline broadcasts

---
 src/you_get/extractors/showroom.py | 33 ++++++++++++++++--------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/showroom.py b/src/you_get/extractors/showroom.py
index d0f636bc01..606dc80666 100644
--- a/src/you_get/extractors/showroom.py
+++ b/src/you_get/extractors/showroom.py
@@ -5,7 +5,7 @@
 from ..common import *
 import urllib.error
 from json import loads
-from time import time
+from time import time, sleep
 
 #----------------------------------------------------------------------
 def showroom_get_roomid_by_room_url_key(room_url_key):
@@ -25,19 +25,22 @@ def showroom_get_roomid_by_room_url_key(room_url_key):
 
 def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    timestamp = str(int(time() * 1000))
-    api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
-    html = get_content(api_endpoint)
-    html = json.loads(html)
-    #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
-    if len(html) < 1:
-        log.wtf('Cannot find any live URL! Maybe the live have ended or haven\'t start yet?')
-        
+    while True:
+        timestamp = str(int(time() * 1000))
+        api_endpoint = 'https://www.showroom-live.com/api/live/streaming_url?room_id={room_id}&_={timestamp}'.format(room_id = room_id, timestamp = timestamp)
+        html = get_content(api_endpoint)
+        html = json.loads(html)
+        #{'streaming_url_list': [{'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 1, 'label': 'original spec(low latency)', 'is_default': True, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed/playlist.m3u8', 'is_default': True, 'id': 2, 'type': 'hls', 'label': 'original spec'}, {'url': 'rtmp://52.197.69.198:1935/liveedge', 'id': 3, 'label': 'low spec(low latency)', 'is_default': False, 'type': 'rtmp', 'stream_name': '7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low'}, {'url': 'http://52.197.69.198:1935/liveedge/7656a6d5baa1d77075c971f6d8b6dc61b979fc913dc5fe7cc1318281793436ed_low/playlist.m3u8', 'is_default': False, 'id': 4, 'type': 'hls', 'label': 'low spec'}]}
+        if len(html) >= 1:
+            break
+        log.w('The live show is currently offline.')
+        sleep(1)
+
     #This is mainly for testing the M3U FFmpeg parser so I would ignore any non-m3u ones
     stream_url = [i['url'] for i in html['streaming_url_list'] if i['is_default'] and i['type'] == 'hls'][0]
-    
+
     assert stream_url
-    
+
     #title
     title = ''
     profile_api = 'https://www.showroom-live.com/api/room/profile?room_id={room_id}'.format(room_id = room_id)
@@ -46,12 +49,12 @@ def showroom_download_by_room_id(room_id, output_dir = '.', merge = False, info_
         title = html['main_name']
     except KeyError:
         title = 'Showroom_{room_id}'.format(room_id = room_id)
-    
+
     type_, ext, size = url_info(stream_url)
     print_info(site_info, title, type_, size)
     if not info_only:
         download_url_ffmpeg(url=stream_url, title=title, ext= 'mp4', output_dir=output_dir)
-    
+
 
 #----------------------------------------------------------------------
 def showroom_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
@@ -59,9 +62,9 @@ def showroom_download(url, output_dir = '.', merge = False, info_only = False, *
     if re.match( r'(\w+)://www.showroom-live.com/([-\w]+)', url):
         room_url_key = match1(url, r'\w+://www.showroom-live.com/([-\w]+)')
         room_id = showroom_get_roomid_by_room_url_key(room_url_key)
-        showroom_download_by_room_id(room_id, output_dir, merge, 
+        showroom_download_by_room_id(room_id, output_dir, merge,
                                     info_only)
 
 site_info = "Showroom"
 download = showroom_download
-download_playlist = playlist_not_supported('showroom')
\ No newline at end of file
+download_playlist = playlist_not_supported('showroom')

From e5c606503cbeb67656155695155da15c91f2d7fe Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 15:48:34 -0400
Subject: [PATCH 0183/1225] add the option to use choco to install on windows

---
 README.md | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index aa7f61a090..73af5dc305 100644
--- a/README.md
+++ b/README.md
@@ -43,7 +43,7 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 ### Prerequisites
 
-The following dependencies are required and must be installed separately, unless you are using a pre-built package on Windows:
+The following dependencies are required and must be installed separately, unless you are using a pre-built package or chocolatey on Windows:
 
 * **[Python 3](https://www.python.org/downloads/)**
 * **[FFmpeg](https://www.ffmpeg.org/)** (strongly recommended) or [Libav](https://libav.org/)
@@ -61,11 +61,19 @@ Add the following line to your `.zshrc`:
 
     antigen bundle soimort/you-get
 
-### Option 3: Use a pre-built package (Windows only)
+### Option 3: Using [Chocolatey](https://chocolatey.org/) (Windows only)
+
+    choco install you-get
+
+The chocolatey package source can be found [here](https://github.com/chantisnake/you-get-choco)
+
+The chocolatey package page can be found [here](https://chocolatey.org/packages/you-get/0.4.486)
+
+### Option 4: Use a pre-built package (Windows only)
 
 Download the `exe` (standalone) or `7z` (all dependencies included) from: <https://github.com/soimort/you-get/releases/latest>.
 
-### Option 4: Download from GitHub
+### Option 5: Download from GitHub
 
 You may either download the [stable](https://github.com/soimort/you-get/archive/master.zip) (identical with the latest release on PyPI) or the [develop](https://github.com/soimort/you-get/archive/develop.zip) (more hotfixes, unstable features) branch of `you-get`. Unzip it, and put the directory containing the `you-get` script into your `PATH`.
 
@@ -83,7 +91,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option 5: Git clone
+### Option6: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 
@@ -93,7 +101,7 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
-### Option 6: Homebrew (Mac only)
+### Option 7: Homebrew (Mac only)
 
 You can install `you-get` easily via:
 
@@ -119,6 +127,12 @@ or download the latest release via:
 $ you-get https://github.com/soimort/you-get/archive/master.zip
 ```
 
+or use [chocolatey package manager](https://chocolatey.org):
+
+```
+> choco upgrade you-get 
+```
+
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
 
 ```

From 06252f0b6ee151944bdd8e29e5feea71d88328c4 Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 15:51:33 -0400
Subject: [PATCH 0184/1225] fix an unfortunate typo

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 73af5dc305..ff85a69df9 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option6: Git clone
+### Option 6: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 

From c43bb6006dabf5e7d2f22bb96fff5058b765dc01 Mon Sep 17 00:00:00 2001
From: Cheng Zhang <13501393281@163.com>
Date: Sun, 10 Jul 2016 20:02:41 -0400
Subject: [PATCH 0185/1225] fix the order

let choco option more close to home brew since they are alike
---
 README.md | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/README.md b/README.md
index ff85a69df9..8afc84ea07 100644
--- a/README.md
+++ b/README.md
@@ -61,19 +61,11 @@ Add the following line to your `.zshrc`:
 
     antigen bundle soimort/you-get
 
-### Option 3: Using [Chocolatey](https://chocolatey.org/) (Windows only)
-
-    choco install you-get
-
-The chocolatey package source can be found [here](https://github.com/chantisnake/you-get-choco)
-
-The chocolatey package page can be found [here](https://chocolatey.org/packages/you-get/0.4.486)
-
-### Option 4: Use a pre-built package (Windows only)
+### Option 3: Use a pre-built package (Windows only)
 
 Download the `exe` (standalone) or `7z` (all dependencies included) from: <https://github.com/soimort/you-get/releases/latest>.
 
-### Option 5: Download from GitHub
+### Option 4: Download from GitHub
 
 You may either download the [stable](https://github.com/soimort/you-get/archive/master.zip) (identical with the latest release on PyPI) or the [develop](https://github.com/soimort/you-get/archive/develop.zip) (more hotfixes, unstable features) branch of `you-get`. Unzip it, and put the directory containing the `you-get` script into your `PATH`.
 
@@ -91,7 +83,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option 6: Git clone
+### Option 5: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 
@@ -101,6 +93,12 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
+### Option 6: Using [Chocolatey](https://chocolatey.org/) (Windows only)
+
+```
+> choco install you-get
+```
+
 ### Option 7: Homebrew (Mac only)
 
 You can install `you-get` easily via:

From d5e801b7c109c7f5b7166947d01ce7630f236cc1 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Sun, 17 Jul 2016 02:24:30 -0400
Subject: [PATCH 0186/1225] [iQiyi]not import json_output module

From: @wwqgtxx , #1282, as requested.
---
 src/you_get/extractors/iqiyi.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 248446a779..80fa5f938d 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -2,14 +2,15 @@
 
 from ..common import *
 from ..extractor import VideoExtractor
+from ..util import log
+from .. import json_output
+
 from uuid import uuid4
 from random import random,randint
 import json
 from math import floor
 from zlib import decompress
 import hashlib
-from ..util import log
-
 import time
 
 '''

From b95b1a10ee01299358fd23e9ef85e1738cdfacd2 Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 12:04:15 +0800
Subject: [PATCH 0187/1225] fix playlist download

---
 src/you_get/extractors/youku.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index fefaf5ee68..e04a51fc14 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -76,7 +76,7 @@ def oset(xs):
         for x in xs:
             if x not in mem:
                 mem.add(x)
-                yield(x)
+        return mem
 
     def get_vid_from_url(url):
         """Extracts video ID from URL.
@@ -89,7 +89,7 @@ def get_vid_from_url(url):
     def get_playlist_id_from_url(url):
         """Extracts playlist ID from URL.
         """
-        return match1(url, r'youku\.com/playlist_show/id_([a-zA-Z0-9=]+)')
+        return match1(url, r'youku\.com/albumlist/show\?id=([a-zA-Z0-9=]+)')
 
     def download_playlist_by_url(self, url, **kwargs):
         self.url = url
@@ -97,16 +97,19 @@ def download_playlist_by_url(self, url, **kwargs):
         try:
             playlist_id = self.__class__.get_playlist_id_from_url(self.url)
             assert playlist_id
-
-            video_page = get_content('http://www.youku.com/playlist_show/id_%s' % playlist_id)
+            video_page = get_content('http://list.youku.com/albumlist/show?id=%s' % playlist_id)
             videos = Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', video_page))
-
             # Parse multi-page playlists
-            for extra_page_url in Youku.oset(re.findall('href="(http://www\.youku\.com/playlist_show/id_%s_[^?"]+)' % playlist_id, video_page)):
-                extra_page = get_content(extra_page_url)
-                videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
-
-        except:
+            last_page_url = re.findall(r'href="(/albumlist/show\?id=%s[^"]+)" title="末页"' % playlist_id, video_page)[0]
+            num_pages = int(re.findall(r'page=([0-9]+)\.htm', last_page_url)[0])
+            if (num_pages > 0):
+                # download one by one
+                for pn in range(1, num_pages + 1):
+                    extra_page_url = re.sub(r'page=([0-9]+)\.htm', r'page=%s.htm' % pn, last_page_url)
+                    extra_page = get_content('http://list.youku.com' + extra_page_url)
+                    videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
+        except Exception as e:
+            print(e)
             # Show full list of episodes
             if match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)'):
                 ep_id = match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)')

From cb2878b8cfd974605112cc6c4fff2c405c92f39f Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 12:09:20 +0800
Subject: [PATCH 0188/1225] avoid unnecessary downloading page 1

---
 src/you_get/extractors/youku.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index e04a51fc14..345347d0dd 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -104,12 +104,11 @@ def download_playlist_by_url(self, url, **kwargs):
             num_pages = int(re.findall(r'page=([0-9]+)\.htm', last_page_url)[0])
             if (num_pages > 0):
                 # download one by one
-                for pn in range(1, num_pages + 1):
+                for pn in range(2, num_pages + 1):
                     extra_page_url = re.sub(r'page=([0-9]+)\.htm', r'page=%s.htm' % pn, last_page_url)
                     extra_page = get_content('http://list.youku.com' + extra_page_url)
                     videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
-        except Exception as e:
-            print(e)
+        except:
             # Show full list of episodes
             if match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)'):
                 ep_id = match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)')

From fdb1809455247746e76aa164f08485bc21de7426 Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Wed, 20 Jul 2016 15:18:28 +0800
Subject: [PATCH 0189/1225] fix socks DNS resolve issue

---
 src/you_get/common.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c8843ddf13..a9b12b085a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1214,15 +1214,18 @@ def version():
 
     if (socks_proxy):
         try:
-          import socket
-          import socks
-          socks_proxy_addrs = socks_proxy.split(':')
-          socks.set_default_proxy(socks.SOCKS5, 
-                                  socks_proxy_addrs[0], 
-                                  int(socks_proxy_addrs[1]))
-          socket.socket = socks.socksocket
+            import socket
+            import socks
+            socks_proxy_addrs = socks_proxy.split(':')
+            socks.set_default_proxy(socks.SOCKS5, 
+                                    socks_proxy_addrs[0], 
+                                    int(socks_proxy_addrs[1]))
+            socket.socket = socks.socksocket
+            def getaddrinfo(*args):
+                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+            socket.getaddrinfo = getaddrinfo
         except ImportError:
-          log.w('Error importing PySocks library, socks proxy ignored.'
+            log.w('Error importing PySocks library, socks proxy ignored.'
                 'In order to use use socks proxy, please install PySocks.')
     else:
         import socket

From 09bfe2847243f58a3e11bfde3ec1bf1fd84ce5ec Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 21 Jul 2016 00:50:21 -0400
Subject: [PATCH 0190/1225] [Processor] Fix rtmpdump not portable under Windows

---
 src/you_get/processor/rtmpdump.py | 32 +++++++++++++++++++++++--------
 1 file changed, 24 insertions(+), 8 deletions(-)

diff --git a/src/you_get/processor/rtmpdump.py b/src/you_get/processor/rtmpdump.py
index cf5f822cf0..f1ab9a4b97 100644
--- a/src/you_get/processor/rtmpdump.py
+++ b/src/you_get/processor/rtmpdump.py
@@ -41,16 +41,32 @@ def download_rtmpdump_stream(url, title, ext,params={},output_dir='.'):
     subprocess.call(cmdline)
     return
 
-#
-#To be refactor
-#To the future myself: Remember to refactor the same function in ffmpeg.py
 #
 def play_rtmpdump_stream(player, url, params={}):
-    cmdline="rtmpdump -r '%s' "%url
+    
+    #construct left side of pipe
+    cmdline = [RTMPDUMP, '-r']
+    cmdline.append(url)
+    
+    #append other params if exist
     for key in params.keys():
-        cmdline+=key+" "+params[key] if params[key]!=None else ""+" "
-    cmdline+=" -o - | %s -"%player
-    print(cmdline)
-    os.system(cmdline)
+        cmdline.append(key)
+        if params[key]!=None:
+            cmdline.append(params[key])
+
+    cmdline.append('-o')
+    cmdline.append('-')
+
+    #pipe start
+    cmdline.append('|')
+    cmdline.append(player)
+    cmdline.append('-')
+
+    #logging
+    print("Call rtmpdump:\n"+" ".join(cmdline)+"\n")
+
+    #call RTMPDump!
+    subprocess.call(cmdline)
+    
     # os.system("rtmpdump -r '%s' -y '%s' -o - | %s -" % (url, playpath, player))
     return

From 80a8265a49a6d5b96d885023d72c3710681d19cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Jul 2016 00:04:39 +0200
Subject: [PATCH 0191/1225] [extractors] import zhanqi (close #1297)

---
 src/you_get/extractors/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index b879b83e36..2f148acab0 100644
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -79,3 +79,4 @@
 from .youtube import *
 from .ted import *
 from .khan import *
+from .zhanqi import *

From fbf5c491aa8f20263c5fa0c36f7bb01c468f8c91 Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Wed, 27 Jul 2016 13:42:06 +0800
Subject: [PATCH 0192/1225] [bilibili] video title may have space around

---
 src/you_get/extractors/bilibili.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 247825988b..5c010a8009 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -125,8 +125,8 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         url = r1(r'"([^"]+)" class="v-av-link"', html)
         html = get_content(url)
 
-    title = r1_of([r'<meta name="title" content="([^<>]{1,999})" />',
-                   r'<h1[^>]*>([^<>]+)</h1>'], html)
+    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
     if title:
         title = unescape_html(title)
         title = escape_file_path(title)
@@ -139,14 +139,14 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     cid = cid.split('&')[0]
     if t == 'cid':
         if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>([^<>]+)</title>', html)
+            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
             bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
 
         else:
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>(.+)</option>', html)
+            titles = re.findall('<option value=.*>\s*(.+)\s*</option>', html)
             for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
@@ -163,7 +163,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # no multi-P
             if not pages:
                 cids = [cid]
-                titles = [r1(r'<option value=.* selected>(.+)</option>', html) or title]
+                titles = [r1(r'<option value=.* selected>\s*(.+)\s*</option>', html) or title]
 
             for i in range(len(cids)):
                 bilibili_download_by_cid(cids[i],

From 056082c36cb3e93f9858ee132ff7e9979a88af96 Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Wed, 27 Jul 2016 14:16:32 +0800
Subject: [PATCH 0193/1225] [bilibili] video title may have space around

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5c010a8009..3fbf946fa0 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -146,7 +146,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # multi-P
             cids = []
             pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*(.+)\s*</option>', html)
+            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
             for i, page in enumerate(pages):
                 html = get_html("http://www.bilibili.com%s" % page)
                 flashvars = r1_of([r'(cid=\d+)',
@@ -163,7 +163,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
             # no multi-P
             if not pages:
                 cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*(.+)\s*</option>', html) or title]
+                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
 
             for i in range(len(cids)):
                 bilibili_download_by_cid(cids[i],

From 8e8506790fb033f166d898d24676fdcf2b57fe82 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 28 Jul 2016 04:23:44 -0400
Subject: [PATCH 0194/1225] [Youku COOP] Add embsig to parasing

---
 src/you_get/extractors/youku.py | 35 ++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 345347d0dd..853a75ba68 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -330,22 +330,51 @@ def extract(self, **kwargs):
 
     def open_download_by_vid(self, client_id, vid, **kwargs):
         """self, str, str, **kwargs->None
+        
+        Arguments:
+        client_id:        An ID per client. For now we only know Acfun's
+                          such ID.
+        
+        vid:              An video ID for each video, starts with "C".
+        
+        kwargs['embsig']: Youku COOP's anti hotlinking.
+                          For Acfun, an API call must be done to Acfun's
+                          server, or the "playsign" of the content of sign_url
+                          shall be empty.
+        
+        Misc:
         Override the original one with VideoExtractor.
-        Most of the credit are to @ERioK, who gave his POC."""
+        
+        Author:
+        Most of the credit are to @ERioK, who gave his POC.
+        
+        History:
+        Jul.28.2016 Youku COOP now have anti hotlinking via embsig. """
         self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
         self.f_code_2 = 'msjv7h2b'
+        
+        # as in VideoExtractor
         self.url = None
         self.vid = vid
         self.name = "优酷开放平台 (Youku COOP)"
 
         #A little bit of work before self.prepare
-        sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+        
+        #Change as Jul.28.2016 Youku COOP updates its platform to add ant hotlinking
+        if kwargs['embsig']:
+            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}&embsig={embsig}".format(client_id = client_id, video_id = vid, embsig = kwargs['embsig'])
+        else:
+            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
+
         playsign = json.loads(get_content(sign_url))['playsign']
-    
+
+        #to be injected and replace ct10 and 12
         api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
         api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
         
         self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
+        
+        #exact copy from original VideoExtractor
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             unset_proxy()
 

From 063d4b1457e132ef2dc1e3be6154f10ee21a5cef Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 28 Jul 2016 04:24:12 -0400
Subject: [PATCH 0195/1225] [Acfun] Update embsig with Youku COOP to bypass
 anti hot linking

---
 src/you_get/extractors/acfun.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 983606d42f..2ccc471878 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -17,10 +17,24 @@ def get_srt_json(id):
     return get_html(url)
 
 def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
+    """str, str, str, bool, bool ->None
+    
+    Download Acfun video by vid.
+    
+    Call Acfun API, decide which site to use, and pass the job to its
+    extractor.
+    """
+
+    #first call the main parasing API
     info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
+
     sourceType = info['sourceType']
+
+    #decide sourceId to know which extractor to use
     if 'sourceId' in info: sourceId = info['sourceId']
     # danmakuId = info['danmakuId']
+
+    #call extractor decided by sourceId
     if sourceType == 'sina':
         sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'youku':
@@ -32,11 +46,13 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     elif sourceType == 'letv':
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
+        #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
+        embsig =  info['encode']
         a = 'http://api.aixifan.com/plays/%s' % vid
         s = json.loads(get_content(a, headers={'deviceType': '2'}))
         if s['data']['source'] == "zhuzhan-youku":
             sourceId = s['data']['sourceId']
-            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir,merge=merge, info_only=info_only, embsig = embsig, **kwargs)
     else:
         raise NotImplementedError(sourceType)
 

From 468fb6f403d98d6870fab4d6670a6da06f5f9238 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Fri, 29 Jul 2016 01:27:22 -0400
Subject: [PATCH 0196/1225] [Processor]FFmpeg: check arguments not None

---
 src/you_get/processor/ffmpeg.py | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c23b0eaf66..deb13bd22c 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -213,10 +213,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
         
     ffmpeg_params = []
     #should these exist...
-    if len(params) > 0:
-        for k, v in params:
-            ffmpeg_params.append(k)
-            ffmpeg_params.append(v)
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
         
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
@@ -249,10 +251,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
 def ffmpeg_play_stream(player, url, params={}):
     ffmpeg_params = []
     #should these exist...
-    if len(params) > 0:
-        for k, v in params:
-            ffmpeg_params.append(k)
-            ffmpeg_params.append(v)
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
         
     print('Playing streaming content with FFmpeg, press 1 to stop recording...')
     ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']

From 6689f45469f5f14dd338ae7743313f7b8c6c5174 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 13:35:05 +0800
Subject: [PATCH 0197/1225] [Douyutv]Update api

Thanks #1299 @cnbeining #1235 @hellofwy
---
 src/you_get/extractors/douyutv.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 449022caa6..29bcedd47e 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -6,27 +6,44 @@
 import json
 import hashlib
 import time
+import random
+import string
+import requests
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     room_id = url[url.rfind('/')+1:]
-    #Thanks to @yan12125 for providing decoding method!!
-    suffix = 'room/%s?aid=android&client_sys=android&time=%d' % (room_id, int(time.time()))
-    sign = hashlib.md5((suffix + '1231').encode('ascii')).hexdigest()
-    json_request_url = "http://www.douyu.com/api/v1/%s&auth=%s" % (suffix, sign)
+
+    json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_html(json_request_url)
     data = json.loads(content)['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
+
     title = data.get('room_name')
     show_status = data.get('show_status')
     if show_status is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
+
+    tt = int(time.time() / 60)
+    did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
+    sign = hashlib.md5((room_id + did + 'A12Svb&%1UUmf@hC' + "%d" % tt).encode("utf-8")).hexdigest()
+    json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
+
+    payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
+    r = requests.post(json_request_url, data=payload)
+    content = r.json()
+    data = content['data']
+
+    server_status = data.get('error',0)
+    if server_status is not 0:
+        raise ValueError("Server returned error:%s" % server_status)
+
     real_url = data.get('rtmp_url')+'/'+data.get('rtmp_live')
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"
 download = douyutv_download

From 06b57e2db7cce08ff0b6f2a5e0ad6d2723f1d017 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 18:25:40 +0800
Subject: [PATCH 0198/1225] fix CI failure by using 3rd packages

---
 src/you_get/extractors/douyutv.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 29bcedd47e..38b2d48d5a 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -8,13 +8,13 @@
 import time
 import random
 import string
-import requests
+import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     room_id = url[url.rfind('/')+1:]
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
-    content = get_html(json_request_url)
+    content = get_content(json_request_url)
     data = json.loads(content)['data']
     server_status = data.get('error',0)
     if server_status is not 0:
@@ -27,14 +27,19 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     tt = int(time.time() / 60)
     did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
-    sign = hashlib.md5((room_id + did + 'A12Svb&%1UUmf@hC' + "%d" % tt).encode("utf-8")).hexdigest()
-    json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
+    sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
+    sign = hashlib.md5(sign_content.encode('utf-8')).hexdigest()
 
+    json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
     payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-    r = requests.post(json_request_url, data=payload)
-    content = r.json()
-    data = content['data']
+    json_data = json.dumps(payload).encode('utf8')
+    postdata = urllib.parse.urlencode(payload)
+    req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
+    with urllib.request.urlopen(req) as response:
+        the_page = response.read()
 
+    content = json.loads(the_page.decode('utf-8'))
+    data = content['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
@@ -43,6 +48,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
+        print(real_url)
         download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"

From ddbc2caa23ed96a38146aa735dd83a48cd855f0a Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 20:50:15 +0800
Subject: [PATCH 0199/1225] add support for non digit channel

event page is still not support because the rule is complicated
---
 src/you_get/extractors/douyutv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 38b2d48d5a..8cdb86252a 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -11,7 +11,9 @@
 import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    room_id = url[url.rfind('/')+1:]
+    html = get_content(url)
+    room_id_patt = r'"room_id"\s*:\s*(\d+),'
+    room_id = match1(html, room_id_patt)
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_content(json_request_url)
@@ -48,7 +50,6 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        print(real_url)
         download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
 
 site_info = "douyu.com"

From 2322a16ca829e110743d037e0fb4a6cc2cfb4799 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Fri, 29 Jul 2016 21:30:43 +0800
Subject: [PATCH 0200/1225] delete useless line

---
 src/you_get/extractors/douyutv.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 8cdb86252a..6587efe9ac 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -34,7 +34,6 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
     payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-    json_data = json.dumps(payload).encode('utf8')
     postdata = urllib.parse.urlencode(payload)
     req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
     with urllib.request.urlopen(req) as response:

From d0215aa23882d0d8c63cfaf856f6e8e7b73d4b1b Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Mon, 1 Aug 2016 05:39:38 -0400
Subject: [PATCH 0201/1225] [Bigthink]Add Bigthink

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/bigthink.py | 76 ++++++++++++++++++++++++++++++
 3 files changed, 78 insertions(+)
 mode change 100644 => 100755 src/you_get/extractors/__init__.py
 create mode 100644 src/you_get/extractors/bigthink.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a9b12b085a..3cbee37381 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -8,6 +8,7 @@
     'baidu'            : 'baidu',
     'bandcamp'         : 'bandcamp',
     'baomihua'         : 'baomihua',
+    'bigthink'         : 'bigthink',
     'bilibili'         : 'bilibili',
     'cctv'             : 'cntv',
     'cntv'             : 'cntv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
old mode 100644
new mode 100755
index 2f148acab0..97ab0b41fb
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -5,6 +5,7 @@
 from .archive import *
 from .baidu import *
 from .bandcamp import *
+from .bigthink import *
 from .bilibili import *
 from .cbs import *
 from .ckplayer import *
diff --git a/src/you_get/extractors/bigthink.py b/src/you_get/extractors/bigthink.py
new file mode 100644
index 0000000000..1dd196d5ec
--- /dev/null
+++ b/src/you_get/extractors/bigthink.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+import json
+
+class Bigthink(VideoExtractor):
+    name = "Bigthink"
+
+    stream_types = [  #this is just a sample. Will make it in prepare()
+        # {'id': '1080'},
+        # {'id': '720'},
+        # {'id': '360'},
+        # {'id': '288'},
+        # {'id': '190'},
+        # {'id': '180'},
+        
+    ]
+
+    @staticmethod
+    def get_streams_by_id(account_number, video_id):
+        """
+        int, int->list
+        
+        Get the height of the videos.
+        
+        Since brightcove is using 3 kinds of links: rtmp, http and https,
+        we will be using the HTTPS one to make it secure.
+        
+        If somehow akamaihd.net is blocked by the Great Fucking Wall,
+        change the "startswith https" to http.
+        """
+        endpoint = 'https://edge.api.brightcove.com/playback/v1/accounts/{account_number}/videos/{video_id}'.format(account_number = account_number, video_id = video_id)
+        fake_header_id = fake_headers
+        #is this somehow related to the time? Magic....
+        fake_header_id['Accept'] ='application/json;pk=BCpkADawqM1cc6wmJQC2tvoXZt4mrB7bFfi6zGt9QnOzprPZcGLE9OMGJwspQwKfuFYuCjAAJ53JdjI8zGFx1ll4rxhYJ255AXH1BQ10rnm34weknpfG-sippyQ'
+
+        html = get_content(endpoint, headers= fake_header_id)
+        html_json = json.loads(html)
+
+        link_list = []
+
+        for i in html_json['sources']:
+            if 'src' in i:  #to avoid KeyError
+                if i['src'].startswith('https'):
+                    link_list.append((str(i['height']), i['src']))
+
+        return link_list
+
+    def prepare(self, **kwargs):
+
+        html = get_content(self.url)
+
+        self.title = match1(html, r'<meta property="og:title" content="([^"]*)"')
+
+        account_number = match1(html, r'data-account="(\d+)"')
+
+        video_id = match1(html, r'data-brightcove-id="(\d+)"')
+        
+        assert account_number, video_id
+
+        link_list = self.get_streams_by_id(account_number, video_id)
+
+        for i in link_list:
+            self.stream_types.append({'id': str(i[0])})
+            self.streams[i[0]] = {'url': i[1]}
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+
+site = Bigthink()
+download = site.download_by_url

From f8939128e6a17304aee641da2f1eb15535693dfb Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 3 Aug 2016 21:39:10 -0400
Subject: [PATCH 0202/1225] [QQ]Add QQ Daxue

---
 src/you_get/extractors/qq.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 783bb2f5a5..80f8d6b7fe 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,6 +5,7 @@
 from ..common import *
 from .qie import download as qieDownload
 from urllib.parse import urlparse,parse_qs
+
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
     content = get_html(api)
@@ -23,7 +24,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     if not info_only:
         download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    """"""
     if 'live.qq.com' in url:
         qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
         return 
@@ -35,7 +38,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         content = get_html(url)
         url = match1(content,r'window\.location\.href="(.*?)"')
         
-    if 'kuaibao.qq.com' in url:
+    if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')

From af039eae5f9418806b3d0297baec599df91c6986 Mon Sep 17 00:00:00 2001
From: johan <johan@ximalaya.com>
Date: Thu, 4 Aug 2016 15:32:22 +0800
Subject: [PATCH 0203/1225] [ifeng] fix video info xml url

---
 src/you_get/extractors/ifeng.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index 75078177d1..20f3874312 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -6,7 +6,7 @@
 
 def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
     assert r1(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', id), id
-    url = 'http://v.ifeng.com/video_info_new/%s/%s/%s.xml' % (id[-2], id[-2:], id)
+    url = 'http://vxml.ifengimg.com/video_info_new/%s/%s/%s.xml' % (id[-2], id[-2:], id)
     xml = get_html(url, 'utf-8')
     title = r1(r'Name="([^"]+)"', xml)
     title = unescape_html(title)
@@ -15,7 +15,7 @@ def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_
     r = randint(10, 19)
     url = url.replace('http://video.ifeng.com/', 'http://video%s.ifeng.com/' % r)
     type, ext, size = url_info(url)
-    
+
     print_info(site_info, title, ext, size)
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge = merge)
@@ -24,7 +24,7 @@ def ifeng_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     id = r1(r'/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.shtml$', url)
     if id:
         return ifeng_download_by_id(id, None, output_dir = output_dir, merge = merge, info_only = info_only)
-    
+
     html = get_html(url)
     id = r1(r'var vid="([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"', html)
     assert id, "can't find video info"

From 021a2f62a3764fc42d141aaa461b74fd7d12c13b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 5 Aug 2016 01:24:12 +0200
Subject: [PATCH 0204/1225] version 0.4.523

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 46850c8a22..88a851417b 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.486'
+__version__ = '0.4.523'

From a8015ec92ebcb0f31036c44eb459a8e2bcdc522c Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Tue, 9 Aug 2016 02:27:59 +0800
Subject: [PATCH 0205/1225] [Douyutv]fix channel on event

some channel on event will redirect to event page
exp. http://www.douyu.com/65251 redirect to http://www.douyu.com/cms/zt/RioOlympics.html#a
if channel is on event, use entering url to get room_id, and non-digital room_id will not work
---
 src/you_get/extractors/douyutv.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 6587efe9ac..61b6d32471 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -6,14 +6,15 @@
 import json
 import hashlib
 import time
-import random
-import string
+import uuid
 import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_content(url)
     room_id_patt = r'"room_id"\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
+    if room_id == "0":
+        room_id = url[url.rfind('/')+1:]
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_content(json_request_url)
@@ -28,7 +29,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
     tt = int(time.time() / 60)
-    did = ''.join([random.choice(string.ascii_uppercase + string.digits) for n in range(32)])
+    did = uuid.uuid4().hex.upper()
     sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
     sign = hashlib.md5(sign_content.encode('utf-8')).hexdigest()
 
@@ -37,10 +38,9 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     postdata = urllib.parse.urlencode(payload)
     req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
     with urllib.request.urlopen(req) as response:
-        the_page = response.read()
+        content = response.read()
 
-    content = json.loads(the_page.decode('utf-8'))
-    data = content['data']
+    data = json.loads(content.decode('utf-8'))['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)

From e765f328057fcf340e580fcfb09ac257928fd295 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 10 Aug 2016 13:56:52 +0200
Subject: [PATCH 0206/1225] [netease] fix #1341, also do not create new
 directory when info_only

---
 src/you_get/extractors/netease.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 332d48ec1d..b37ee00323 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -4,6 +4,7 @@
 __all__ = ['netease_download']
 
 from ..common import *
+from ..util import fs
 from json import loads
 import hashlib
 import base64
@@ -28,10 +29,10 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
 
         artist_name = j['album']['artists'][0]['name']
         album_name = j['album']['name']
-        new_dir = output_dir + '/' + "%s - %s" % (artist_name, album_name)
-        if not os.path.exists(new_dir):
-            os.mkdir(new_dir)
+        new_dir = output_dir + '/' + fs.legitimize("%s - %s" % (artist_name, album_name))
         if not info_only:
+            if not os.path.exists(new_dir):
+                os.mkdir(new_dir)
             cover_url = j['album']['picUrl']
             download_urls([cover_url], "cover", "jpg", 0, new_dir)
 
@@ -46,10 +47,10 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
     elif "playlist" in url:
         j = loads(get_content("http://music.163.com/api/playlist/detail?id=%s&csrf_token=" % rid, headers={"Referer": "http://music.163.com/"}))
 
-        new_dir = output_dir + '/' + j['result']['name']
-        if not os.path.exists(new_dir):
-            os.mkdir(new_dir)
+        new_dir = output_dir + '/' + fs.legitimize(j['result']['name'])
         if not info_only:
+            if not os.path.exists(new_dir):
+                os.mkdir(new_dir)
             cover_url = j['result']['coverImgUrl']
             download_urls([cover_url], "cover", "jpg", 0, new_dir)
 
@@ -61,7 +62,7 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
                 netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
             except: pass
 
-    elif "song" in url: 
+    elif "song" in url:
         j = loads(get_content("http://music.163.com/api/song/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_song_download(j["songs"][0], output_dir=output_dir, info_only=info_only)
         try: # download lyrics

From 0c50e0466650b184c1af15fdcc822d1f6489a09a Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 11 Aug 2016 03:29:59 -0400
Subject: [PATCH 0207/1225] [DouyuTV] Fix discarding output_dir

---
 src/you_get/extractors/douyutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 6587efe9ac..dd6aa12217 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -49,7 +49,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', None, output_dir = output_dir, merge = merge)
 
 site_info = "douyu.com"
 download = douyutv_download

From bb485a111074a6571c10d7e26e8ed048847eddb6 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Thu, 11 Aug 2016 03:30:21 -0400
Subject: [PATCH 0208/1225] [ffmpeg] Fix wrong format of output dir

---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index deb13bd22c..96e1348be9 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -209,7 +209,7 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     output = title + '.' + ext
     
     if not (output_dir == '.'):
-        output = output_dir + output
+        output = output_dir + '/' + output
         
     ffmpeg_params = []
     #should these exist...

From 3a8ce8819df02a090dcf6b041ea0e1f49faf48a4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 14 Aug 2016 03:52:02 +0200
Subject: [PATCH 0209/1225] [ffmpeg] remove unuseful ffmpeg_play_stream(), fix
 #1336

---
 src/you_get/common.py           | 11 +++----
 src/you_get/processor/ffmpeg.py | 54 +++++----------------------------
 2 files changed, 13 insertions(+), 52 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 3cbee37381..2ff61d5561 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -910,8 +910,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
         return
 
     if player:
-        from .processor.ffmpeg import ffmpeg_play_stream
-        ffmpeg_play_stream(player, url, params)
+        launch_player(player, [url])
         return
 
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
@@ -1218,12 +1217,12 @@ def version():
             import socket
             import socks
             socks_proxy_addrs = socks_proxy.split(':')
-            socks.set_default_proxy(socks.SOCKS5, 
-                                    socks_proxy_addrs[0], 
+            socks.set_default_proxy(socks.SOCKS5,
+                                    socks_proxy_addrs[0],
                                     int(socks_proxy_addrs[1]))
             socket.socket = socks.socksocket
-            def getaddrinfo(*args):
-                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+            def getaddrinfo(*args):
+                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
             socket.getaddrinfo = getaddrinfo
         except ImportError:
             log.w('Error importing PySocks library, socks proxy ignored.'
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 96e1348be9..c34f43c28a 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -204,13 +204,13 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     """str, str->True
     WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
     You can basicly download anything with this function
-    but better leave it alone with 
+    but better leave it alone with
     """
     output = title + '.' + ext
-    
+
     if not (output_dir == '.'):
         output = output_dir + '/' + output
-        
+
     ffmpeg_params = []
     #should these exist...
     if params is not None:
@@ -219,58 +219,20 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
                 ffmpeg_params.append(k)
                 ffmpeg_params.append(v)
 
-        
+
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
-    
+
     if FFMPEG == 'avconv':  #who cares?
         ffmpeg_params += ['-c', 'copy', output]
     else:
         ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
-    
-    ffmpeg_params.append(output)
-    
-    print(' '.join(ffmpeg_params))
-    
-    try:
-        a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
-        a.communicate()
-    except KeyboardInterrupt:
-        try:
-            a.stdin.write('q'.encode('utf-8'))
-        except:
-            pass
 
-    return True
-
-#
-#To be refactor
-#Direct copy of rtmpdump.py
-#
-def ffmpeg_play_stream(player, url, params={}):
-    ffmpeg_params = []
-    #should these exist...
-    if params is not None:
-        if len(params) > 0:
-            for k, v in params:
-                ffmpeg_params.append(k)
-                ffmpeg_params.append(v)
+    ffmpeg_params.append(output)
 
-        
-    print('Playing streaming content with FFmpeg, press 1 to stop recording...')
-    ffmpeg_params = [FFMPEG] + LOGLEVEL + ['-y', '-re', '-i']
-    ffmpeg_params.append(url)  #not the same here!!!!
-    
-    if FFMPEG == 'avconv':  #who cares?
-        ffmpeg_params += ['-c', 'copy', '|']
-    else:
-        ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', '|']
-    
-    ffmpeg_params += [player, '-']
-    
     print(' '.join(ffmpeg_params))
-    
+
     try:
         a = subprocess.Popen(ffmpeg_params, stdin= subprocess.PIPE)
         a.communicate()
@@ -280,4 +242,4 @@ def ffmpeg_play_stream(player, url, params={}):
         except:
             pass
 
-    return True
\ No newline at end of file
+    return True

From 326bae23e15e1508dd4d1398f2e746ccdafdec41 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 18:54:58 +0200
Subject: [PATCH 0210/1225] [qq] fix #1355

---
 src/you_get/extractors/qq.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 80f8d6b7fe..59460e1da9 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -29,7 +29,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
     if 'live.qq.com' in url:
         qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
-        return 
+        return
 
     #do redirect
     if 'v.qq.com/page' in url:
@@ -37,7 +37,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         # http://v.qq.com/page/k/9/7/k0194pwgw97.html
         content = get_html(url)
         url = match1(content,r'window\.location\.href="(.*?)"')
-        
+
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_html(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
@@ -49,10 +49,11 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = vid
     else:
         content = get_html(url)
-        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5 
+        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
         vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
+        title = match1(content, r'"title":"([^"]+)"') if not title else title
         title = vid if not title else title #general fallback
 
 

From 7cecd0368dea3e8e7318f48908e39d3d8405c95f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 21:44:05 +0200
Subject: [PATCH 0211/1225] [bilibili] close #1340

---
 src/you_get/extractors/bilibili.py | 206 +++++++----------------------
 1 file changed, 48 insertions(+), 158 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 3fbf946fa0..c7c4fac91b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -2,94 +2,17 @@
 
 __all__ = ['bilibili_download']
 
-from ..common import *
-
-from .sina import sina_download_by_vid
-from .tudou import tudou_download_by_id
-from .youku import youku_download_by_vid
-
-import hashlib
+import json
 import re
+from ..common import *
 
-appkey = 'f3bb208b3d081dc8'
-
-
-def get_srt_xml(id):
-    url = 'http://comment.bilibili.com/%s.xml' % id
-    return get_html(url)
-
-
-def parse_srt_p(p):
-    fields = p.split(',')
-    assert len(fields) == 8, fields
-    time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
-    time = float(time)
-
-    mode = int(mode)
-    assert 1 <= mode <= 8
-    # mode 1~3: scrolling
-    # mode 4: bottom
-    # mode 5: top
-    # mode 6: reverse?
-    # mode 7: position
-    # mode 8: advanced
-
-    pool = int(pool)
-    assert 0 <= pool <= 2
-    # pool 0: normal
-    # pool 1: srt
-    # pool 2: special?
-
-    font_size = int(font_size)
-
-    font_color = '#%06x' % int(font_color)
-
-    return pool, mode, font_size, font_color
-
-
-def parse_srt_xml(xml):
-    d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
-    for x, y in d:
-        p = parse_srt_p(x)
-    raise NotImplementedError()
-
-
-def parse_cid_playurl(xml):
-    from xml.dom.minidom import parseString
-    try:
-        doc = parseString(xml.encode('utf-8'))
-        urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
-        return urls
-    except:
-        return []
-
-
-def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
-    urls = []
-    for cid in cids:
-        url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
-        urls += [i
-                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                 for i in parse_cid_playurl(get_content(url))]
-
-    type_ = ''
-    size = 0
-    for url in urls:
-        _, type_, temp = url_info(url)
-        size += temp
-
-    print_info(site_info, title, type_, size)
-    if not info_only:
-        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
+def get_srt_xml(cid):
+    return get_html('http://comment.bilibili.com/%s.xml' % cid)
 
-def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    url = 'http://interface.bilibili.com/playurl?appkey=' + appkey + '&cid=' + cid
-    urls = [i
-            if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-            else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-            for i in parse_cid_playurl(get_content(url))]
+def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    title = r1(r'cid=(\d+)', url)
+    info = json.loads(get_content(url))
+    urls = [i['url'] for i in info['durl']]
 
     type_ = ''
     size = 0
@@ -104,82 +27,50 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
-
-def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
-    urls = parse_cid_playurl(get_content(api_url))
-
-    for url in urls:
-        _, type_, _ = url_info(url)
-        size = 0
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_content(url)
-
-    if re.match(r'https?://bangumi\.bilibili\.com/', url):
-        # quick hack for bangumi URLs
-        url = r1(r'"([^"]+)" class="v-av-link"', html)
-        html = get_content(url)
-
-    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
-                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
-    if title:
-        title = unescape_html(title)
-        title = escape_file_path(title)
-
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
-                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-    assert flashvars
-    flashvars = flashvars.replace(': ', '=')
-    t, cid = flashvars.split('=', 1)
-    cid = cid.split('&')[0]
-    if t == 'cid':
-        if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
-            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+    if re.match(r'https?://interface\.bilibili\.com/', url):
+        # quick hack for explicit API
+        bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
 
+    html = get_content(url)
+    main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                        r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
+    cid = r1(r'cid=(\d+)', html)
+
+    aid = r1(r'av(\d+)', url)
+    page = r1(r'index_(\d+)', url)
+    sub_titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+    if page is None and sub_titles: # download all
+        for t in enumerate(sub_titles):
+            page, sub_title = t[0] + 1, t[1]
+            title = main_title + ' - ' + sub_title
+
+            api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
+            info = json.loads(get_content(api))
+            src = info['src']
+            _, type_, size = url_info(src)
+            print_info(site_info, title, type_, size)
+            if not info_only:
+                download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
+
+    else: # download selected
+        if page is None: page = 1
+        sub_title = r1('<option value=.* selected>\s*([^<>]+)\s*</option>', html)
+        if sub_title is None:
+            sub_title = r1('<option value=.*>\s*([^<>]+)\s*</option>', html)
+        if sub_title:
+            title = main_title + ' - ' + sub_title
         else:
-            # multi-P
-            cids = []
-            pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-            for i, page in enumerate(pages):
-                html = get_html("http://www.bilibili.com%s" % page)
-                flashvars = r1_of([r'(cid=\d+)',
-                                   r'flashvars="([^"]+)"',
-                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-                if flashvars:
-                    t, cid = flashvars.split('=', 1)
-                    cids.append(cid.split('&')[0])
-                if url.endswith(page):
-                    cids = [cid.split('&')[0]]
-                    titles = [titles[i]]
-                    break
+            title = main_title
 
-            # no multi-P
-            if not pages:
-                cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-
-            for i in range(len(cids)):
-                bilibili_download_by_cid(cids[i],
-                                         titles[i],
-                                         output_dir=output_dir,
-                                         merge=merge,
-                                         info_only=info_only)
-
-    elif t == 'vid':
-        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'ykid':
-        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'uid':
-        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-    else:
-        raise NotImplementedError(flashvars)
+        api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
+        info = json.loads(get_content(api))
+        src = info['src']
+        _, type_, size = url_info(src)
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
 
     if not info_only and not dry_run:
         if not kwargs['caption']:
@@ -191,7 +82,6 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
-
 site_info = "bilibili.com"
 download = bilibili_download
 download_playlist = bilibili_download

From 550ef5aeb24bfc799a23091afc60ebbba7808793 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Aug 2016 23:35:33 +0200
Subject: [PATCH 0212/1225] [acfun] fix #1353 (get and only get the specified
 part)

---
 src/you_get/extractors/acfun.py | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 2ccc471878..5c6dc987a0 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -18,9 +18,9 @@ def get_srt_json(id):
 
 def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
     """str, str, str, bool, bool ->None
-    
+
     Download Acfun video by vid.
-    
+
     Call Acfun API, decide which site to use, and pass the job to its
     extractor.
     """
@@ -78,15 +78,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = escape_file_path(title)
     assert title
 
-    videos = re.findall("data-vid=\"(\d+)\".*href=\"[^\"]+\".*title=\"([^\"]+)\"", html)
-    for video in videos:
-        p_vid = video[0]
-        p_title = title + " - " + video[1] if video[1] != '删除标签' else title
-        acfun_download_by_vid(p_vid, p_title,
-                              output_dir=output_dir,
-                              merge=merge,
-                              info_only=info_only,
-                              **kwargs)
+    video = re.search('data-vid="(\d+)"\s*data-scode="".*>([^<]+)</a>', html)
+    vid = video.group(1)
+    title = title + ' - ' + video.group(2)
+    acfun_download_by_vid(vid, title,
+                          output_dir=output_dir,
+                          merge=merge,
+                          info_only=info_only,
+                          **kwargs)
 
 site_info = "AcFun.tv"
 download = acfun_download

From ce759590d3d72de226b6b1e692be86d658742ac0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 22 Aug 2016 11:41:26 +0200
Subject: [PATCH 0213/1225] [ifeng] fix #1358

---
 src/you_get/extractors/ifeng.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index 20f3874312..6ca45b30eb 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -13,7 +13,7 @@ def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_
     url = r1(r'VideoPlayUrl="([^"]+)"', xml)
     from random import randint
     r = randint(10, 19)
-    url = url.replace('http://video.ifeng.com/', 'http://video%s.ifeng.com/' % r)
+    url = url.replace('http://wideo.ifeng.com/', 'http://ips.ifeng.com/wideo.ifeng.com/')
     type, ext, size = url_info(url)
 
     print_info(site_info, title, ext, size)

From f8bf3e6cf3c96786523180d0568c72e0aa1d332e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 22 Aug 2016 23:58:48 +0200
Subject: [PATCH 0214/1225] [sina] fix #1361, close #1332

---
 src/you_get/extractors/sina.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/sina.py b/src/you_get/extractors/sina.py
index 88f819c0d1..fd45833570 100644
--- a/src/you_get/extractors/sina.py
+++ b/src/you_get/extractors/sina.py
@@ -14,7 +14,7 @@ def get_k(vid, rand):
 
 def video_info_xml(vid):
     rand = "0.{0}{1}".format(randint(10000, 10000000), randint(10000, 10000000))
-    url = 'http://v.iask.com/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
+    url = 'http://ask.ivideo.sina.com.cn/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
     xml = get_content(url, headers=fake_headers, decoded=True)
     return xml
 
@@ -71,7 +71,7 @@ def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             vid = vids[-1]
 
     if vid is None:
-        vid = match1(video_page, r'vid:(\d+)')
+        vid = match1(video_page, r'vid:"?(\d+)"?')
     if vid:
         title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
         sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)

From 16015a2db0575756e24856d1e17865e01e6b73c4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 23 Aug 2016 00:54:33 +0200
Subject: [PATCH 0215/1225] version 0.4.536

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 88a851417b..a877863a17 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.523'
+__version__ = '0.4.536'

From 315634332a0303297905deaf27e6779ac38a1cf6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 23 Aug 2016 14:39:43 +0200
Subject: [PATCH 0216/1225] [twitter] fix #1357

---
 src/you_get/extractors/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 09956141b9..db01daa26c 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -73,10 +73,10 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             scribe_playlist_url = i['scribe_playlist_url']
             return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)
 
-        if source.endswith('.mp4'):
-            urls = [source]
-        else:
+        try:
             urls = extract_m3u(source)
+        except:
+            urls = [source]
         size = urls_size(urls)
         mime, ext = 'video/mp4', 'mp4'
 

From a3f3d880f9126ce003d94fe4105ca11e5862e127 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Aug 2016 02:04:05 +0200
Subject: [PATCH 0217/1225] [bilibili] quick hack for bangumi URLs again (fix
 #1362)

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c7c4fac91b..a9caf8ed27 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -33,6 +33,11 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return
 
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        html = get_content(url)
+        url = r1(r'"([^"]+)" class="v-av-link"', html)
+
     html = get_content(url)
     main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
                         r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)

From ebd3db9acb34f40e84f75b81db00ad25573b7602 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 24 Aug 2016 23:00:48 -0400
Subject: [PATCH 0218/1225] [BokeCC] Add support

---
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/bokecc.py   | 95 ++++++++++++++++++++++++++++++
 2 files changed, 96 insertions(+)
 create mode 100644 src/you_get/extractors/bokecc.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 97ab0b41fb..d283e30cf1 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -7,6 +7,7 @@
 from .bandcamp import *
 from .bigthink import *
 from .bilibili import *
+from .bokecc import *
 from .cbs import *
 from .ckplayer import *
 from .cntv import *
diff --git a/src/you_get/extractors/bokecc.py b/src/you_get/extractors/bokecc.py
new file mode 100644
index 0000000000..8566e8282f
--- /dev/null
+++ b/src/you_get/extractors/bokecc.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+import xml.etree.ElementTree as ET
+
+class BokeCC(VideoExtractor):
+    name = "BokeCC"
+
+    stream_types = [  # we do now know for now, as we have to check the
+                      # output from the API
+    ]
+
+    API_ENDPOINT = 'http://p.bokecc.com/'
+
+
+    def download_by_id(self, vid = '', title = None, output_dir='.', merge=True, info_only=False,**kwargs):
+        """self, str->None
+        
+        Keyword arguments:
+        self: self
+        vid: The video ID for BokeCC cloud, something like
+        FE3BB999594978049C33DC5901307461
+        
+        Calls the prepare() to download the video.
+        
+        If no title is provided, this method shall try to find a proper title
+        with the information providin within the
+        returned content of the API."""
+
+        assert vid
+
+        self.prepare(vid = vid, title = title, **kwargs)
+
+        self.extract(**kwargs)
+
+        self.download(output_dir = output_dir, 
+                    merge = merge, 
+                    info_only = info_only, **kwargs)
+
+    def prepare(self, vid = '', title = None, **kwargs):
+        assert vid
+
+        api_url = self.API_ENDPOINT + \
+            'servlet/playinfo?vid={vid}&m=0'.format(vid = vid)  #return XML
+
+        html = get_content(api_url)
+        self.tree = ET.ElementTree(ET.fromstring(html))
+
+        if self.tree.find('result').text != '1':
+            log.wtf('API result says failed!')
+            raise 
+
+        if title is None:
+            self.title = '_'.join([i.text for i in tree.iterfind('video/videomarks/videomark/markdesc')])
+        else:
+            self.title = title
+
+        for i in self.tree.iterfind('video/quality'):
+            quality = i.attrib ['value']
+            url = i[0].attrib['playurl']
+            self.stream_types.append({'id': quality,
+                                      'video_profile': i.attrib ['desp']})
+            self.streams[quality] = {'url': url,
+                                     'video_profile': i.attrib ['desp']}
+            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+
+
+    def extract(self, **kwargs):
+        for i in self.streams:
+            s = self.streams[i]
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # Extract the stream
+            stream_id = kwargs['stream_id']
+
+            if stream_id not in self.streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # Extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+            _, s['container'], s['size'] = url_info(s['url'])
+            s['src'] = [s['url']]
+
+site = BokeCC()
+
+# I don't know how to call the player directly so I just put it here
+# just in case anyone touchs it -- Beining@Aug.24.2016
+#download = site.download_by_url
+#download_playlist = site.download_by_url
+
+bokecc_download_by_id = site.download_by_id

From e60981f91e070c222923ea68a7a6aff79a650e5f Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Wed, 24 Aug 2016 23:12:06 -0400
Subject: [PATCH 0219/1225] [Wanmen]Add support

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/wanmen.py   | 123 +++++++++++++++++++++++++++++
 3 files changed, 125 insertions(+)
 create mode 100755 src/you_get/extractors/wanmen.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2ff61d5561..a5a0fbab63 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -79,6 +79,7 @@
     'videomega'        : 'videomega',
     'vidto'            : 'vidto',
     'vimeo'            : 'vimeo',
+    'wanmen'           : 'wanmen',
     'weibo'            : 'miaopai',
     'veoh'             : 'veoh',
     'vine'             : 'vine',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index d283e30cf1..e69bc2fd01 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -74,6 +74,7 @@
 from .vine import *
 from .vk import *
 from .w56 import *
+from .wanmen import *
 from .xiami import *
 from .yinyuetai import *
 from .yixia import *
diff --git a/src/you_get/extractors/wanmen.py b/src/you_get/extractors/wanmen.py
new file mode 100755
index 0000000000..20c543c1e5
--- /dev/null
+++ b/src/you_get/extractors/wanmen.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+
+__all__ = ['wanmen_download', 'wanmen_download_by_course', 'wanmen_download_by_course_topic', 'wanmen_download_by_course_topic_part']
+
+from ..common import *
+from .bokecc import bokecc_download_by_id
+from json import loads
+
+
+##Helper functions
+def _wanmen_get_json_api_content_by_courseID(courseID):
+    """int->JSON
+    
+    Return a parsed JSON tree of WanMen's API."""
+
+    return loads(get_content('http://api.wanmen.org/course/getCourseNested/{courseID}'.format(courseID = courseID)))
+
+def _wanmen_get_title_by_json_topic_part(json_content, tIndex, pIndex):
+    """JSON, int, int, int->str
+    
+    Get a proper title with courseid+topicID+partID."""
+
+    return '_'.join([json_content[0]['name'],
+                    json_content[0]['Topics'][tIndex]['name'],
+                    json_content[0]['Topics'][tIndex]['Parts'][pIndex]['name']])
+
+
+def _wanmen_get_boke_id_by_json_topic_part(json_content, tIndex, pIndex):
+    """JSON, int, int, int->str
+    
+    Get one BokeCC video ID with courseid+topicID+partID."""
+
+    return json_content[0]['Topics'][tIndex]['Parts'][pIndex]['ccVideoLink']
+
+
+##Parsers
+def wanmen_download_by_course(json_api_content, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int->None
+    
+    Download a WHOLE course.
+    Reuse the API call to save time."""
+
+    for tIndex in range(len(json_api_content[0]['Topics'])):
+        for pIndex in range(len(json_api_content[0]['Topics'][tIndex]['Parts'])):
+            wanmen_download_by_course_topic_part(json_api_content,
+                                                 tIndex,
+                                                 pIndex,
+                                                 output_dir=output_dir,
+                                                 merge=merge,
+                                                 info_only=info_only,
+                                                 **kwargs)
+
+
+def wanmen_download_by_course_topic(json_api_content, tIndex, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int, int->None
+    
+    Download a TOPIC of a course.
+    Reuse the API call to save time."""
+
+    for pIndex in range(len(json_api_content[0]['Topics'][tIndex]['Parts'])):
+        wanmen_download_by_course_topic_part(json_api_content,
+                                             tIndex,
+                                             pIndex, 
+                                            output_dir=output_dir,
+                                            merge=merge,
+                                            info_only=info_only,
+                                            **kwargs)
+
+def wanmen_download_by_course_topic_part(json_api_content, tIndex, pIndex, output_dir='.', merge=True, info_only=False, **kwargs):
+    """int, int, int->None
+    
+    Download ONE PART of the course."""
+
+    html = json_api_content
+
+    title = _wanmen_get_title_by_json_topic_part(html, 
+                                                  tIndex, 
+                                                  pIndex)
+
+    bokeccID = _wanmen_get_boke_id_by_json_topic_part(html,
+                                                      tIndex, 
+                                                     pIndex)
+
+    bokecc_download_by_id(vid = bokeccID, title = title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
+
+##Main entrance
+def wanmen_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+
+    if not 'wanmen.org' in url:
+        log.wtf('You are at the wrong place dude. This is for WanMen University!')
+        raise
+
+    courseID = int(match1(url, r'course\/(\d+)'))
+    assert courseID > 0  #without courseID we cannot do anything
+
+    tIndex = int(match1(url, r'tIndex=(\d+)'))
+
+    pIndex = int(match1(url, r'pIndex=(\d+)'))
+
+    json_api_content = _wanmen_get_json_api_content_by_courseID(courseID)
+
+    if pIndex:  #only download ONE single part
+        assert tIndex >= 0
+        wanmen_download_by_course_topic_part(json_api_content, tIndex, pIndex, 
+                                            output_dir = output_dir, 
+                                            merge = merge, 
+                                            info_only = info_only)
+    elif tIndex:  #download a topic
+        wanmen_download_by_course_topic(json_api_content, tIndex, 
+                                       output_dir = output_dir, 
+                                       merge = merge, 
+                                       info_only = info_only)
+    else:  #download the whole course
+        wanmen_download_by_course(json_api_content,
+                                 output_dir = output_dir, 
+                                 merge = merge, 
+                                 info_only = info_only)
+
+
+site_info = "WanMen University"
+download = wanmen_download
+download_playlist = wanmen_download_by_course
\ No newline at end of file

From 6fc2cc375ea4bb2dcb6bab74ce67688af0efde59 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Aug 2016 19:27:40 +0200
Subject: [PATCH 0220/1225] [universal] call embed_download only if
 content_type is text/html (#1369)

---
 src/you_get/extractors/universal.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index b0d929c9ec..ebab70f88f 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -6,20 +6,20 @@
 from .embed import *
 
 def universal_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    try:
-        embed_download(url, output_dir, merge=merge, info_only=info_only)
-    except: pass
-    else: return
+    content_type = get_head(url, headers=fake_headers)['Content-Type']
+    if content_type.startswith('text/html'):
+        try:
+            embed_download(url, output_dir, merge=merge, info_only=info_only)
+        except: pass
+        else: return
 
     domains = url.split('/')[2].split('.')
     if len(domains) > 2: domains = domains[1:]
     site_info = '.'.join(domains)
 
-    response = get_response(url, faker=True)
-    content_type = response.headers['Content-Type']
-
     if content_type.startswith('text/html'):
         # extract an HTML page
+        response = get_response(url, faker=True)
         page = str(response.data)
 
         page_title = r1(r'<title>([^<]*)', page)

From d66749801f6fbb0aee4e2030e93448e9d46e13f8 Mon Sep 17 00:00:00 2001
From: David Zhuang <david.zhuang@mail.utoronto.ca>
Date: Sun, 28 Aug 2016 23:39:19 -0400
Subject: [PATCH 0221/1225] [Bilibili] Change back to extract via interface API
 with Miniloader s key

---
 src/you_get/extractors/bilibili.py | 207 ++++++++++++++++++++++-------
 1 file changed, 157 insertions(+), 50 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a9caf8ed27..e9d3e7ad02 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -2,17 +2,96 @@
 
 __all__ = ['bilibili_download']
 
-import json
-import re
 from ..common import *
 
-def get_srt_xml(cid):
-    return get_html('http://comment.bilibili.com/%s.xml' % cid)
+from .sina import sina_download_by_vid
+from .tudou import tudou_download_by_id
+from .youku import youku_download_by_vid
+
+import hashlib
+import re
+
+appkey = 'f3bb208b3d081dc8'
+SECRETKEY_MINILOADER = '1c15888dc316e05a15fdd0a02ed6584f'
+
+def get_srt_xml(id):
+    url = 'http://comment.bilibili.com/%s.xml' % id
+    return get_html(url)
+
+
+def parse_srt_p(p):
+    fields = p.split(',')
+    assert len(fields) == 8, fields
+    time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
+    time = float(time)
+
+    mode = int(mode)
+    assert 1 <= mode <= 8
+    # mode 1~3: scrolling
+    # mode 4: bottom
+    # mode 5: top
+    # mode 6: reverse?
+    # mode 7: position
+    # mode 8: advanced
+
+    pool = int(pool)
+    assert 0 <= pool <= 2
+    # pool 0: normal
+    # pool 1: srt
+    # pool 2: special?
+
+    font_size = int(font_size)
+
+    font_color = '#%06x' % int(font_color)
+
+    return pool, mode, font_size, font_color
+
+
+def parse_srt_xml(xml):
+    d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
+    for x, y in d:
+        p = parse_srt_p(x)
+    raise NotImplementedError()
+
+
+def parse_cid_playurl(xml):
+    from xml.dom.minidom import parseString
+    try:
+        doc = parseString(xml.encode('utf-8'))
+        urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
+        return urls
+    except:
+        return []
+
+
+def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
+    urls = []
+    for cid in cids:
+        sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
+        url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
+        urls += [i
+                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+                 for i in parse_cid_playurl(get_content(url))]
+
+    type_ = ''
+    size = 0
+    for url in urls:
+        _, type_, temp = url_info(url)
+        size += temp
+
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
 
-def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    title = r1(r'cid=(\d+)', url)
-    info = json.loads(get_content(url))
-    urls = [i['url'] for i in info['durl']]
+def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
+    sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
+    url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
+    urls = [i
+            if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+            else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+            for i in parse_cid_playurl(get_content(url))]
 
     type_ = ''
     size = 0
@@ -27,55 +106,82 @@ def bilibili_download_by_api(url, output_dir='.', merge=True, info_only=False, *
     if not info_only:
         download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
 
+
+def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
+    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
+    urls = parse_cid_playurl(get_content(api_url))
+
+    for url in urls:
+        _, type_, _ = url_info(url)
+        size = 0
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
+
+
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    if re.match(r'https?://interface\.bilibili\.com/', url):
-        # quick hack for explicit API
-        bilibili_download_by_api(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
-        return
+    html = get_content(url)
 
     if re.match(r'https?://bangumi\.bilibili\.com/', url):
         # quick hack for bangumi URLs
-        html = get_content(url)
         url = r1(r'"([^"]+)" class="v-av-link"', html)
+        html = get_content(url)
+
+    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
+                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
+    if title:
+        title = unescape_html(title)
+        title = escape_file_path(title)
+
+    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+    assert flashvars
+    flashvars = flashvars.replace(': ', '=')
+    t, cid = flashvars.split('=', 1)
+    cid = cid.split('&')[0]
+    if t == 'cid':
+        if re.match(r'https?://live\.bilibili\.com/', url):
+            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
+            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-    html = get_content(url)
-    main_title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
-                        r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
-    cid = r1(r'cid=(\d+)', html)
-
-    aid = r1(r'av(\d+)', url)
-    page = r1(r'index_(\d+)', url)
-    sub_titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-    if page is None and sub_titles: # download all
-        for t in enumerate(sub_titles):
-            page, sub_title = t[0] + 1, t[1]
-            title = main_title + ' - ' + sub_title
-
-            api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
-            info = json.loads(get_content(api))
-            src = info['src']
-            _, type_, size = url_info(src)
-            print_info(site_info, title, type_, size)
-            if not info_only:
-                download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
-
-    else: # download selected
-        if page is None: page = 1
-        sub_title = r1('<option value=.* selected>\s*([^<>]+)\s*</option>', html)
-        if sub_title is None:
-            sub_title = r1('<option value=.*>\s*([^<>]+)\s*</option>', html)
-        if sub_title:
-            title = main_title + ' - ' + sub_title
         else:
-            title = main_title
+            # multi-P
+            cids = []
+            pages = re.findall('<option value=\'([^\']*)\'', html)
+            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+            for i, page in enumerate(pages):
+                html = get_html("http://www.bilibili.com%s" % page)
+                flashvars = r1_of([r'(cid=\d+)',
+                                   r'flashvars="([^"]+)"',
+                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+                if flashvars:
+                    t, cid = flashvars.split('=', 1)
+                    cids.append(cid.split('&')[0])
+                if url.endswith(page):
+                    cids = [cid.split('&')[0]]
+                    titles = [titles[i]]
+                    break
 
-        api = 'http://www.bilibili.com/m/html5?aid=%s&page=%s' % (aid, page)
-        info = json.loads(get_content(api))
-        src = info['src']
-        _, type_, size = url_info(src)
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([src], title, type_, total_size=size, output_dir=output_dir, merge=merge)
+            # no multi-P
+            if not pages:
+                cids = [cid]
+                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
+
+            for i in range(len(cids)):
+                bilibili_download_by_cid(cids[i],
+                                         titles[i],
+                                         output_dir=output_dir,
+                                         merge=merge,
+                                         info_only=info_only)
+
+    elif t == 'vid':
+        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif t == 'ykid':
+        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif t == 'uid':
+        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+    else:
+        raise NotImplementedError(flashvars)
 
     if not info_only and not dry_run:
         if not kwargs['caption']:
@@ -87,6 +193,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
             x.write(xml)
 
+
 site_info = "bilibili.com"
 download = bilibili_download
-download_playlist = bilibili_download
+download_playlist = bilibili_download
\ No newline at end of file

From 5e612e870ce4cd9971675fb5420cb4b07620c28b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 30 Aug 2016 12:04:12 +0200
Subject: [PATCH 0222/1225] [pptv] fix #447 (per discussion in #1381)

---
 src/you_get/extractors/pptv.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index f9da64a51d..b4ae78d1e3 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -37,14 +37,14 @@ def getkey(s):
             l4+=1
         return l3
         pass
-    
+
     def rot(k,b): ##>>> in as3
         if k>=0:
             return k>>b
         elif k<0:
             return (2**32+k)>>b
         pass
-     
+
     def lot(k,b):
         return (k<<b)%(2**32)
 
@@ -72,10 +72,10 @@ def encrypt(arg1,arg2):
         l18=ord(l8[l12+5])<<8
         l19=ord(l8[l12+6])<<16
         l20=ord(l8[l12+7])<<24
-     
+
         l21=(((0|l13)|l14)|l15)|l16
         l22=(((0|l17)|l18)|l19)|l20
-     
+
         l23=0
         l24=0
         while l24<32:
@@ -91,9 +91,9 @@ def encrypt(arg1,arg2):
             l40=(l39+l7)%(2**32)
             l41=((l37^l38)%(2**32)^l40)%(2**32)
             l22=(l22+l41)%(2**32)
-     
+
             l24+=1
-     
+
         l11+=chr(rot(l21,0)&0xff)
         l11+=chr(rot(l21,8)&0xff)
         l11+=chr(rot(l21,16)&0xff)
@@ -102,9 +102,9 @@ def encrypt(arg1,arg2):
         l11+=chr(rot(l22,8)&0xff)
         l11+=chr(rot(l22,16)&0xff)
         l11+=chr(rot(l22,24)&0xff)
-     
-        return l11        
-            
+
+        return l11
+
 
     loc1=hex(int(arg))[2:]+(16-len(hex(int(arg))[2:]))*"\x00"
     SERVER_KEY="qqqqqww"+"\x00"*9
@@ -129,7 +129,7 @@ def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 
     pieces = re.findall('<sgm no="(\d+)"[^<>]+fs="(\d+)"', xml)
     numbers, fs = zip(*pieces)
-    urls=[ "http://ccf.pptv.com/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(i,rid,key,k)  for i in range(max(map(int,numbers))+1)]
+    urls=["http://{}/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(host,i,rid,key,k) for i in range(max(map(int,numbers))+1)]
 
     total_size = sum(map(int, fs))
     assert rid.endswith('.mp4')

From 44c09c542f4e15192283cc34ee201b878b292e82 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 30 Aug 2016 12:42:50 +0200
Subject: [PATCH 0223/1225] [funshion] fix (partly) #1304 and close #1378

---
 src/you_get/extractors/funshion.py | 35 ++++++++++++++----------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
index e50b4bd735..6a24b46402 100644
--- a/src/you_get/extractors/funshion.py
+++ b/src/you_get/extractors/funshion.py
@@ -10,9 +10,9 @@
 def funshion_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     """"""
     if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):  #single video
-        funshion_download_by_url(url, output_dir = '.', merge = False, info_only = False)
-    elif re.match(r'http://www.fun.tv/vplay/g-(\w+)', url):  #whole drama
-        funshion_download_by_drama_url(url, output_dir = '.', merge = False, info_only = False)
+        funshion_download_by_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif re.match(r'http://www.fun.tv/vplay/.*g-(\w+)', url):  #whole drama
+        funshion_download_by_drama_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
         return
 
@@ -25,7 +25,7 @@ def funshion_download_by_url(url, output_dir = '.', merge = False, info_only = F
     if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):
         match = re.search(r'http://www.fun.tv/vplay/v-(\d+)(.?)', url)
     vid = match.group(1)
-    funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = False)
+    funshion_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
 
 #----------------------------------------------------------------------
 def funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = False):
@@ -34,11 +34,11 @@ def funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = F
     """
     title = funshion_get_title_by_vid(vid)
     url_list = funshion_vid_to_urls(vid)
-    
+
     for url in url_list:
         type, ext, size = url_info(url)
         print_info(site_info, title, type, size)
-    
+
     if not info_only:
         download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
@@ -63,14 +63,11 @@ def funshion_download_by_drama_url(url, output_dir = '.', merge = False, info_on
     """str->None
     url = 'http://www.fun.tv/vplay/g-95785/'
     """
-    if re.match(r'http://www.fun.tv/vplay/g-(\w+)', url):
-        match = re.search(r'http://www.fun.tv/vplay/g-(\d+)(.?)', url)
-    id = match.group(1)
-    
+    id = r1(r'http://www.fun.tv/vplay/.*g-(\d+)', url)
     video_list = funshion_drama_id_to_vid(id)
-    
+
     for video in video_list:
-        funshion_download_by_id((video[0], id), output_dir = '.', merge = False, info_only = False)
+        funshion_download_by_id((video[0], id), output_dir=output_dir, merge=merge, info_only=info_only)
         # id is for drama, vid not the same as the ones used in single video
 
 #----------------------------------------------------------------------
@@ -81,11 +78,11 @@ def funshion_download_by_id(vid_id_tuple, output_dir = '.', merge = False, info_
     (vid, id) = vid_id_tuple
     title = funshion_get_title_by_id(vid, id)
     url_list = funshion_id_to_urls(vid)
-    
+
     for url in url_list:
         type, ext, size = url_info(url)
         print_info(site_info, title, type, size)
-    
+
     if not info_only:
         download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
@@ -95,7 +92,7 @@ def funshion_drama_id_to_vid(episode_id):
     id: 95785
     ->[('626464', '1'), ('626466', '2'), ('626468', '3'),...
     Drama ID to vids used in drama.
-    
+
     **THIS VID IS NOT THE SAME WITH THE ONES USED IN SINGLE VIDEO!!**
     """
     html = get_content('http://pm.funshion.com/v5/media/episode?id={episode_id}&cl=aphone&uc=5'.format(episode_id = episode_id))
@@ -118,7 +115,7 @@ def funshion_get_title_by_id(single_episode_id, drama_id):
     Get title for single drama video."""
     html = get_content('http://pm.funshion.com/v5/media/episode?id={id}&cl=aphone&uc=5'.format(id = drama_id))
     c = json.loads(html)
-    
+
     for i in c['episodes']:
         if i['id'] == str(single_episode_id):
             return c['name'] + ' - ' + i['name']
@@ -127,11 +124,11 @@ def funshion_get_title_by_id(single_episode_id, drama_id):
 #----------------------------------------------------------------------
 def select_url_from_video_api(html):
     """str(html)->str(url)
-    
+
     Choose the best one.
-    
+
     Used in both single and drama download.
-    
+
     code definition:
     {'tv': 'liuchang',
     'dvd': 'biaoqing',

From 2baeed762ae90f8f322f687859fcf6dac1d1ff9e Mon Sep 17 00:00:00 2001
From: "H.G" <helloworld@outlook.co.nz>
Date: Mon, 5 Sep 2016 10:50:58 +1200
Subject: [PATCH 0224/1225] [universal] fix issue: "flush" is not available in
 Python under 3.3

---
 src/you_get/common.py             | 16 ++++++++++++++++
 src/you_get/extractor.py          |  1 +
 src/you_get/extractors/iqiyi.py   |  1 +
 src/you_get/extractors/netease.py |  1 +
 src/you_get/processor/ffmpeg.py   |  1 +
 5 files changed, 20 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a5a0fbab63..7f76aaacaf 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1046,6 +1046,22 @@ def set_http_proxy(proxy):
     opener = request.build_opener(proxy_support)
     request.install_opener(opener)
 
+def print_more_compatible(*args, **kwargs):
+    import builtins as __builtin__
+    """Overload default print function as py (<3.3) does not support 'flush' keyword.
+    Although the function name can be same as print to get itself overloaded automatically,
+    I'd rather leave it with a different name and only overload it when importing to make less confusion. """
+    # nothing happens on py3.3 and later
+    if sys.version_info[:2] >= (3, 3):
+        return __builtin__.print(*args, **kwargs)
+
+    # in lower pyver (e.g. 3.2.x), remove 'flush' keyword and flush it as requested
+    doFlush = kwargs.pop('flush', False)
+    ret = __builtin__.print(*args, **kwargs)
+    if doFlush:
+        kwargs.get('file', sys.stdout).flush()
+    return ret
+
 
 
 def download_main(download, download_playlist, urls, playlist, **kwargs):
diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 3cc78289b0..594b908eef 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy
+from .common import print_more_compatible as print
 from .util import log
 from . import json_output
 import os
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 80fa5f938d..91329708a2 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 from ..common import *
+from ..common import print_more_compatible as print
 from ..extractor import VideoExtractor
 from ..util import log
 from .. import json_output
diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index b37ee00323..63ee59b8a8 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -4,6 +4,7 @@
 __all__ = ['netease_download']
 
 from ..common import *
+from ..common import print_more_compatible as print
 from ..util import fs
 from json import loads
 import hashlib
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c34f43c28a..1c0ba1a3de 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -3,6 +3,7 @@
 import os.path
 import subprocess
 from ..util.strings import parameterize
+from ..common import print_more_compatible as print
 
 def get_usable_ffmpeg(cmd):
     try:

From 24b2b4cb55119ab0198c3bc5538854883d794828 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 7 Sep 2016 10:44:05 +0200
Subject: [PATCH 0225/1225] [bilibili] do not hide any URLError (better for
 debugging)

---
 src/you_get/extractors/bilibili.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e9d3e7ad02..c18290b8be 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -95,12 +95,9 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
 
     type_ = ''
     size = 0
-    try:
-        for url in urls:
-            _, type_, temp = url_info(url)
-            size += temp or 0
-    except error.URLError:
-        log.wtf('[Failed] DNS not resolved. Please change your DNS server settings.')
+    for url in urls:
+        _, type_, temp = url_info(url)
+        size += temp or 0
 
     print_info(site_info, title, type_, size)
     if not info_only:
@@ -196,4 +193,4 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
 
 site_info = "bilibili.com"
 download = bilibili_download
-download_playlist = bilibili_download
\ No newline at end of file
+download_playlist = bilibili_download

From 47011101a940f2884aaa1a41c3a72ac7e6f37cda Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Wed, 7 Sep 2016 10:47:38 +0200
Subject: [PATCH 0226/1225] added fix for nonstandard plflags on panda.tv

---
 src/you_get/extractors/panda.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
index 74064f9fcf..3f9ceadec1 100644
--- a/src/you_get/extractors/panda.py
+++ b/src/you_get/extractors/panda.py
@@ -18,10 +18,11 @@ def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     data = json.loads(content)['data']
     title = data.get('roominfo')['name']
     room_key = data.get('videoinfo')['room_key']
+    plflag = data.get('videoinfo')['plflag'].split('_')
     status = data.get('videoinfo')['status']
     if status is not "2":
         raise ValueError("The live stream is not online! (status:%s)" % status)
-    real_url = 'http://pl3.live.panda.tv/live_panda/{}.flv'.format(room_key)
+    real_url = 'http://pl{}.live.panda.tv/live_panda/{}.flv'.format(plflag[1],room_key)
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From 7baa1f029e6c79a5741c07e428930defefd767db Mon Sep 17 00:00:00 2001
From: zhangchao <zhangchaolhr@qq.com>
Date: Mon, 29 Aug 2016 19:14:29 +0800
Subject: [PATCH 0227/1225] qq compatible with new urls and modify baomihua
 getvidourl address

---
 src/you_get/extractors/baomihua.py | 2 +-
 src/you_get/extractors/qq.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
index f8be6fa91a..4c4febb71a 100644
--- a/src/you_get/extractors/baomihua.py
+++ b/src/you_get/extractors/baomihua.py
@@ -7,7 +7,7 @@
 import urllib
 
 def baomihua_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s' % id)
+    html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s&devicetype=phone_app' % id)
     host = r1(r'host=([^&]*)', html)
     assert host
     type = r1(r'videofiletype=([^&]*)', html)
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 59460e1da9..9ca8af8275 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -50,7 +50,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     else:
         content = get_html(url)
         vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = vid[0] if vid else match1(content, r'vid\s*:\s*"\s*([^"]+)"') #general fallback
+        vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
         title = match1(content, r'"title":"([^"]+)"') if not title else title

From 0984190f93bd0b5c55748c41ca657d1ba6bf5a6b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 14 Sep 2016 19:59:55 +0200
Subject: [PATCH 0228/1225] [acfun] improve regex for getting video part title

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 5c6dc987a0..4638cb8f61 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -78,7 +78,7 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = escape_file_path(title)
     assert title
 
-    video = re.search('data-vid="(\d+)"\s*data-scode="".*>([^<]+)</a>', html)
+    video = re.search('data-vid="(\d+)"\s*data-scode=""[^<]*title="([^"]+)"', html)
     vid = video.group(1)
     title = title + ' - ' + video.group(2)
     acfun_download_by_vid(vid, title,

From 38ba0dbe48ecac4b7a354e4cf5766cf9415fb3c9 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Mon, 19 Sep 2016 13:15:25 +0200
Subject: [PATCH 0229/1225] fix for #1405

---
 src/you_get/extractors/youku.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 853a75ba68..1fb09e8cbf 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -314,6 +314,9 @@ def extract(self, **kwargs):
                                 q         = q
                             )
                         ksegs += [i['server'] for i in json.loads(get_content(u))]
+
+                        if (parse_host(ksegs[len(ksegs)-1])[0] == "vali.cp31.ott.cibntv.net"):
+                            ksegs.pop(len(ksegs)-1)
             except error.HTTPError as e:
                 # Use fallback stream data in case of HTTP 404
                 log.e('[Error] ' + str(e))

From 7e5abc937153791ea66b7fc8233ab8e78aa85e80 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 23 Sep 2016 08:39:25 +0200
Subject: [PATCH 0230/1225] version 0.4.555

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index a877863a17..6d91656c9d 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.536'
+__version__ = '0.4.555'

From 8674f43c36055fd1ceb06844a3bb445637e12271 Mon Sep 17 00:00:00 2001
From: zhangchao <zhangchaolhr@qq.com>
Date: Tue, 27 Sep 2016 23:04:01 +0800
Subject: [PATCH 0231/1225] fix bug about renaming videos when downloading with
 ffmpeg

---
 src/you_get/common.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7f76aaacaf..c2b585a656 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -916,6 +916,11 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
 
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
     assert has_ffmpeg_installed(), "FFmpeg not installed."
+    global output_filename
+    if(output_filename)
+        dotPos = output_filename.rfind(".")
+        title = output_filename[:dotPos]
+        ext = output_filename[dotPos+1:]
     ffmpeg_download_stream(url, title, ext, params, output_dir)
 
 def playlist_not_supported(name):

From fb97d7d0c3a4b5c4db28b2f08dee82faa914b7c4 Mon Sep 17 00:00:00 2001
From: Bochun Bai <bbc@sinofool.net>
Date: Mon, 3 Oct 2016 00:18:50 +0800
Subject: [PATCH 0232/1225] Tencent Video use best quality

Prefer 1080p and 720p if available
---
 src/you_get/extractors/qq.py | 75 ++++++++++++++++++++++++++++--------
 1 file changed, 60 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 9ca8af8275..f170752713 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -7,22 +7,67 @@
 from urllib.parse import urlparse,parse_qs
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    api = "http://h5vv.video.qq.com/getinfo?otype=json&platform=10901&vid=%s" % vid
-    content = get_html(api)
-    output_json = json.loads(match1(content, r'QZOutputJson=(.*)')[:-1])
-    url = output_json['vl']['vi'][0]['ul']['ui'][0]['url']
-    fvkey = output_json['vl']['vi'][0]['fvkey']
-    mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
-    if mp4:
-        mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
-    else:
-        mp4 = output_json['vl']['vi'][0]['fn']
-    url = '%s/%s?vkey=%s' % ( url, mp4, fvkey )
-    _, ext, size = url_info(url, faker=True)
+    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3%2E2%2E19%2E333&platform=11&defnpayver=1&vid=' + vid
+    info = get_html(info_api)
+    video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
+    parts_vid = video_json['vl']['vi'][0]['vid']
+    parts_ti = video_json['vl']['vi'][0]['ti']
+    parts_prefix = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
+    parts_formats = video_json['fl']['fi']
+    # find best quality
+    # only looking for fhd(1080p) and shd(720p) here.
+    # 480p usually come with a single file, will be downloaded as fallback.
+    best_quality = ''
+    for part_format in parts_formats:
+        if part_format['name'] == 'fhd':
+            best_quality = 'fhd'
+            break
+
+        if part_format['name'] == 'shd':
+            best_quality = 'shd'
+
+    for part_format in parts_formats:
+        if (not best_quality == '') and (not part_format['name'] == best_quality):
+            continue
+        part_format_id = part_format['id']
+        part_format_sl = part_format['sl']
+        if part_format_sl == 0:
+            part_urls= []
+            total_size = 0
+            try:
+                # For fhd(1080p), every part is about 100M and 6 minutes
+                # try 100 parts here limited download longest single video of 10 hours.
+                for part in range(1,100):
+                    filename = vid + '.p' + str(part_format_id % 1000) + '.' + str(part) + '.mp4'
+                    key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format=%s&vid=%s&filename=%s" % (part_format_id, parts_vid, filename)
+                    #print(filename)
+                    #print(key_api)
+                    part_info = get_html(key_api)
+                    key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
+                    #print(key_json)
+                    vkey = key_json['key']
+                    url = '%s/%s?vkey=%s' % (parts_prefix, filename, vkey)
+                    part_urls.append(url)
+                    _, ext, size = url_info(url, faker=True)
+                    total_size += size
+            except:
+                pass
+            print_info(site_info, parts_ti, ext, total_size)
+            if not info_only:
+                download_urls(part_urls, parts_ti, ext, total_size, output_dir=output_dir, merge=merge)
+        else:
+            fvkey = output_json['vl']['vi'][0]['fvkey']
+            mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
+            if mp4:
+                mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
+            else:
+                mp4 = output_json['vl']['vi'][0]['fn']
+            url = '%s/%s?vkey=%s' % ( parts_prefix, mp4, fvkey )
+            _, ext, size = url_info(url, faker=True)
 
-    print_info(site_info, title, ext, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):

From 3b3e5cfe38fde46afe0ebf2717802c44e8028706 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Oct 2016 14:38:02 +0200
Subject: [PATCH 0233/1225] update README (close #1422)

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index a99e57fddf..1b6533087e 100644
--- a/README.md
+++ b/README.md
@@ -339,6 +339,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Tumblr**  | <https://www.tumblr.com/>     |✓|✓|✓|
 | TED         | <http://www.ted.com/>         |✓| | |
 | SoundCloud  | <https://soundcloud.com/>     | | |✓|
+| SHOWROOM    | <https://www.showroom-live.com/> |✓| | |
 | Pinterest   | <https://www.pinterest.com/>  | |✓| |
 | MusicPlayOn | <http://en.musicplayon.com/>  |✓| | |
 | MTV81       | <http://www.mtv81.com/>       |✓| | |

From c9ffae970e3ebc6131c4b4a6593320ab6fe65675 Mon Sep 17 00:00:00 2001
From: Vicent Tsai <vayn@vayn.de>
Date: Mon, 10 Oct 2016 00:13:01 +0800
Subject: [PATCH 0234/1225] [AcFun] fix #1429

---
 src/you_get/extractors/acfun.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 4638cb8f61..87e005fb91 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -73,14 +73,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     assert re.match(r'http://[^\.]+.acfun.[^\.]+/\D/\D\D(\d+)', url)
     html = get_html(url)
 
-    title = r1(r'<h1 id="txt-title-view">([^<>]+)<', html)
+    title = r1(r'data-title="([^"]+)"', html)
     title = unescape_html(title)
     title = escape_file_path(title)
     assert title
 
-    video = re.search('data-vid="(\d+)"\s*data-scode=""[^<]*title="([^"]+)"', html)
-    vid = video.group(1)
-    title = title + ' - ' + video.group(2)
+    vid = r1('data-vid="(\d+)"', html)
+    up = r1('data-name="([^"]+)"', html)
+    title = title + ' - ' + up
     acfun_download_by_vid(vid, title,
                           output_dir=output_dir,
                           merge=merge,

From 21fc4d4a0999e70b7886d3abd0cfaa6e4244dbae Mon Sep 17 00:00:00 2001
From: chinat <chinaxiahaifeng@gmail.com>
Date: Mon, 10 Oct 2016 11:28:45 +0800
Subject: [PATCH 0235/1225] translate char | to - in filename for ffmpeg concat

---
 src/you_get/util/fs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
index 36e0b29d5b..4f415bf070 100644
--- a/src/you_get/util/fs.py
+++ b/src/you_get/util/fs.py
@@ -10,6 +10,7 @@ def legitimize(text, os=platform.system()):
     text = text.translate({
         0: None,
         ord('/'): '-',
+        ord('|'): '-',
     })
 
     if os == 'Windows':
@@ -20,7 +21,6 @@ def legitimize(text, os=platform.system()):
             ord('*'): '-',
             ord('?'): '-',
             ord('\\'): '-',
-            ord('|'): '-',
             ord('\"'): '\'',
             # Reserved in Windows VFAT
             ord('+'): '-',

From e2f86641ab5354fa103f29709a21ffdb4cf574f8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 14 Oct 2016 20:32:05 +0200
Subject: [PATCH 0236/1225] update README: use https

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 1b6533087e..b994ebd10b 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ Interested? [Install it](#installation) now and [get started by examples](#getti
 
 Are you a Python programmer? Then check out [the source](https://github.com/soimort/you-get) and fork it!
 
-![](http://i.imgur.com/GfthFAz.png)
+![](https://i.imgur.com/GfthFAz.png)
 
 ## Installation
 

From 669d7b558655f4ffa530ad24573936f90119ced2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 19 Oct 2016 20:47:17 +0200
Subject: [PATCH 0237/1225] [youtube] unescape HTML entities, fix #1462

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 0ef390ed2d..33e3923e67 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -236,7 +236,7 @@ def prepare(self, **kwargs):
                     start = '{:0>2}:{:0>2}:{:06.3f}'.format(int(h), int(m), s).replace('.', ',')
                     m, s = divmod(finish, 60); h, m = divmod(m, 60)
                     finish = '{:0>2}:{:0>2}:{:06.3f}'.format(int(h), int(m), s).replace('.', ',')
-                    content = text.firstChild.nodeValue
+                    content = unescape_html(text.firstChild.nodeValue)
 
                     srt += '%s\n' % str(seq)
                     srt += '%s --> %s\n' % (start, finish)

From 95a8d1e8afdd6df60b3e87ac739cf836b0d0d837 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 20 Oct 2016 14:19:45 +0800
Subject: [PATCH 0238/1225] [huomaotv] add huomao.com suppport

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/huomaotv.py | 36 ++++++++++++++++++++++++++++++
 2 files changed, 37 insertions(+)
 create mode 100644 src/you_get/extractors/huomaotv.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7f76aaacaf..3a60bf12ec 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -27,6 +27,7 @@
     'google'           : 'google',
     'heavy-music'      : 'heavymusic',
     'huaban'           : 'huaban',
+    'huomao'           : 'huomaotv',
     'iask'             : 'sina',
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
diff --git a/src/you_get/extractors/huomaotv.py b/src/you_get/extractors/huomaotv.py
new file mode 100644
index 0000000000..4852ff06f3
--- /dev/null
+++ b/src/you_get/extractors/huomaotv.py
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+
+__all__ = ['huomaotv_download']
+
+from ..common import *
+
+
+def get_mobile_room_url(room_id):
+    return 'http://www.huomao.com/mobile/mob_live?cid=%s' % room_id
+
+
+def get_m3u8_url(stream_id):
+    return 'http://live-ws.huomaotv.cn/live/%s/playlist.m3u8' % stream_id
+
+
+def huomaotv_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    room_id_pattern = r'huomao.com/(\d+)'
+    room_id = match1(url, room_id_pattern)
+    html = get_content(get_mobile_room_url(room_id))
+
+    stream_id_pattern = r'id="html_stream" value="(\w+)"'
+    stream_id = match1(html, stream_id_pattern)
+
+    m3u8_url = get_m3u8_url(stream_id)
+
+    title = match1(html, r'<title>([^<]{1,9999})</title>')
+
+    print_info(site_info, title, 'm3u8', float('inf'))
+
+    if not info_only:
+        download_url_ffmpeg(m3u8_url, title, 'm3u8', None, output_dir=output_dir, merge=merge)
+
+
+site_info = 'huomao.com'
+download = huomaotv_download
+download_playlist = playlist_not_supported('huomao')

From 78fa1161310216e102f84f950b01d43c08899550 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Fri, 14 Oct 2016 00:03:56 -0600
Subject: [PATCH 0239/1225] Add POST method to common.py

---
 src/you_get/common.py | 39 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6c65bd4936..3e1527323c 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -327,6 +327,45 @@ def get_content(url, headers={}, decoded=True):
 
     return data
 
+def post_content(url, headers={}, post_data={}, decoded=True):
+    """Post the content of a URL via sending a HTTP POST request.
+
+    Args:
+        url: A URL.
+        headers: Request headers used by the client.
+        decoded: Whether decode the response body using UTF-8 or the charset specified in Content-Type.
+
+    Returns:
+        The content as a string.
+    """
+
+    logging.debug('post_content: %s \n post_data: %s' % (url, post_data))
+
+    req = request.Request(url, headers=headers)
+    if cookies:
+        cookies.add_cookie_header(req)
+        req.headers.update(req.unredirected_hdrs)
+    post_data_enc = bytes(parse.urlencode(post_data), 'utf-8')
+    response = request.urlopen(req, data = post_data_enc)
+    data = response.read()
+
+    # Handle HTTP compression for gzip and deflate (zlib)
+    content_encoding = response.getheader('Content-Encoding')
+    if content_encoding == 'gzip':
+        data = ungzip(data)
+    elif content_encoding == 'deflate':
+        data = undeflate(data)
+
+    # Decode the response body
+    if decoded:
+        charset = match1(response.getheader('Content-Type'), r'charset=([\w-]+)')
+        if charset is not None:
+            data = data.decode(charset)
+        else:
+            data = data.decode('utf-8')
+
+    return data
+
 def url_size(url, faker = False, headers = {}):
     if faker:
         response = request.urlopen(request.Request(url, headers = fake_headers), None)

From aef17dcb9926f9e2d056a92796dcf58537a42d63 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Fri, 14 Oct 2016 00:03:56 -0600
Subject: [PATCH 0240/1225] Add Baidu Cloud support

---
 src/you_get/extractors/baidu.py | 225 ++++++++++++++++++++++++++++----
 1 file changed, 197 insertions(+), 28 deletions(-)
 mode change 100755 => 100644 src/you_get/extractors/baidu.py

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
old mode 100755
new mode 100644
index aa9caa0c15..d5efaf0bde
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -7,8 +7,10 @@
 from .embed import *
 from .universal import *
 
+
 def baidu_get_song_data(sid):
-    data = json.loads(get_html('http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker = True))['data']
+    data = json.loads(get_html(
+        'http://music.baidu.com/data/music/fmlink?songIds=%s' % sid, faker=True))['data']
 
     if data['xcode'] != '':
         # inside china mainland
@@ -17,22 +19,28 @@ def baidu_get_song_data(sid):
         # outside china mainland
         return None
 
+
 def baidu_get_song_url(data):
     return data['songLink']
 
+
 def baidu_get_song_artist(data):
     return data['artistName']
 
+
 def baidu_get_song_album(data):
     return data['albumName']
 
+
 def baidu_get_song_title(data):
     return data['songName']
 
+
 def baidu_get_song_lyric(data):
     lrc = data['lrcLink']
     return None if lrc is '' else "http://music.baidu.com%s" % lrc
 
+
 def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     data = baidu_get_song_data(sid)
     if data is not None:
@@ -51,7 +59,8 @@ def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     type, ext, size = url_info(url, faker=True)
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([url], file_name, ext, size, output_dir, merge=merge, faker=True)
+        download_urls([url], file_name, ext, size,
+                      output_dir, merge=merge, faker=True)
 
     try:
         type, ext, size = url_info(lrc, faker=True)
@@ -61,12 +70,14 @@ def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
     except:
         pass
 
-def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False):
-    html = get_html('http://music.baidu.com/album/%s' % aid, faker = True)
+
+def baidu_download_album(aid, output_dir='.', merge=True, info_only=False):
+    html = get_html('http://music.baidu.com/album/%s' % aid, faker=True)
     album_name = r1(r'<h2 class="album-name">(.+?)<\/h2>', html)
     artist = r1(r'<span class="author_list" title="(.+?)">', html)
     output_dir = '%s/%s - %s' % (output_dir, artist, album_name)
-    ids = json.loads(r1(r'<span class="album-add" data-adddata=\'(.+?)\'>', html).replace('&quot', '').replace(';', '"'))['ids']
+    ids = json.loads(r1(r'<span class="album-add" data-adddata=\'(.+?)\'>',
+                        html).replace('&quot', '').replace(';', '"'))['ids']
     track_nr = 1
     for id in ids:
         song_data = baidu_get_song_data(id)
@@ -75,38 +86,29 @@ def baidu_download_album(aid, output_dir = '.', merge = True, info_only = False)
         song_lrc = baidu_get_song_lyric(song_data)
         file_name = '%02d.%s' % (track_nr, song_title)
 
-        type, ext, size = url_info(song_url, faker = True)
+        type, ext, size = url_info(song_url, faker=True)
         print_info(site_info, song_title, type, size)
         if not info_only:
-            download_urls([song_url], file_name, ext, size, output_dir, merge = merge, faker = True)
+            download_urls([song_url], file_name, ext, size,
+                          output_dir, merge=merge, faker=True)
 
         if song_lrc:
-            type, ext, size = url_info(song_lrc, faker = True)
+            type, ext, size = url_info(song_lrc, faker=True)
             print_info(site_info, song_title, type, size)
             if not info_only:
-                download_urls([song_lrc], file_name, ext, size, output_dir, faker = True)
+                download_urls([song_lrc], file_name, ext,
+                              size, output_dir, faker=True)
 
         track_nr += 1
 
-def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, **kwargs):
-    if re.match(r'http://imgsrc.baidu.com', url):
-        universal_download(url, output_dir, merge=merge, info_only=info_only)
-        return
-
-    elif re.match(r'http://pan.baidu.com', url):
-        html = get_html(url)
-
-        title = r1(r'server_filename="([^"]+)"', html)
-        if len(title.split('.')) > 1:
-            title = ".".join(title.split('.')[:-1])
 
-        real_url = r1(r'\\"dlink\\":\\"([^"]*)\\"', html).replace('\\\\/', '/')
-        type, ext, size = url_info(real_url, faker = True)
+def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
 
-        print_info(site_info, title, ext, size)
+    if re.match(r'http://pan.baidu.com', url):
+        real_url, title, ext, size = baidu_pan_download(url)
         if not info_only:
-            download_urls([real_url], title, ext, size, output_dir, merge = merge)
-
+            download_urls([real_url], title, ext, size,
+                          output_dir, url, merge=merge, faker=True)
     elif re.match(r'http://music.baidu.com/album/\d+', url):
         id = r1(r'http://music.baidu.com/album/(\d+)', url)
         baidu_download_album(id, output_dir, merge, info_only)
@@ -124,17 +126,20 @@ def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            items = re.findall(r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
+            items = re.findall(
+                r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
             urls = ['http://imgsrc.baidu.com/forum/pic/item/' + i
                     for i in set(items)]
 
             # handle albums
             kw = r1(r'kw=([^&]+)', html) or r1(r"kw:'([^']+)'", html)
             tid = r1(r'tid=(\d+)', html) or r1(r"tid:'([^']+)'", html)
-            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s' % (kw, tid)
+            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s' % (
+                kw, tid)
             album_info = json.loads(get_content(album_url))
             for i in album_info['data']['pic_list']:
-                urls.append('http://imgsrc.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
+                urls.append(
+                    'http://imgsrc.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
 
             ext = 'jpg'
             size = float('Inf')
@@ -144,6 +149,170 @@ def baidu_download(url, output_dir = '.', stream_type = None, merge = True, info
                 download_urls(urls, title, ext, size,
                               output_dir=output_dir, merge=False)
 
+
+def baidu_pan_download(url):
+    errno_patt = r'errno":([^"]+),'
+    refer_url = ""
+    fake_headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'Host': 'pan.baidu.com',
+        'Origin': 'http://pan.baidu.com',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2500.0 Safari/537.36',
+        'Referer': refer_url
+    }
+    if cookies:
+        print('Use user specified cookies')
+    else:
+        print('Generating cookies...')
+        fake_headers['Cookie'] = baidu_pan_gen_cookies(url)
+    refer_url = "http://pan.baidu.com"
+    html = get_content(url, fake_headers, decoded=True)
+    isprotected = False
+    sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+        html)
+    if sign == None:
+        if re.findall(r'\baccess-code\b', html):
+            isprotected = True
+            sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk = baidu_pan_protected_share(
+                url)
+            # raise NotImplementedError("Password required!")
+        if isprotected != True:
+            raise AssertionError("Share not found or canceled: %s" % url)
+    if bdstoken == None:
+        bdstoken = ""
+    if isprotected != True:
+        sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+            html)
+    request_url = "http://pan.baidu.com/api/sharedownload?sign=%s&timestamp=%s&bdstoken=%s&channel=chunlei&clienttype=0&web=1&app_id=%s" % (
+        sign, timestamp, bdstoken, appid)
+    refer_url = url
+    post_data = {
+        'encrypt': 0,
+        'product': 'share',
+        'uk': uk,
+        'primaryid': primary_id,
+        'fid_list': '[' + fs_id + ']'
+    }
+    if isprotected == True:
+        post_data['sekey'] = psk
+    response_content = post_content(request_url, fake_headers, post_data, True)
+    errno = match1(response_content, errno_patt)
+    if errno != "0":
+        raise AssertionError(
+            "Server refused to provide download link! (Errno:%s)" % errno)
+    real_url = r1(r'dlink":"([^"]+)"', response_content).replace('\\/', '/')
+    title = r1(r'server_filename":"([^"]+)"', response_content)
+    assert real_url
+    type, ext, size = url_info(real_url, faker=True)
+    title_wrapped = json.loads('{"wrapper":"%s"}' % title)
+    title = title_wrapped['wrapper']
+    logging.debug(real_url)
+    print_info(site_info, title, ext, size)
+    print('Hold on...')
+    time.sleep(5)
+    return real_url, title, ext, size
+
+
+def baidu_pan_parse(html):
+    sign_patt = r'sign":"([^"]+)"'
+    timestamp_patt = r'timestamp":([^"]+),'
+    appid_patt = r'app_id":"([^"]+)"'
+    bdstoken_patt = r'bdstoken":"([^"]+)"'
+    fs_id_patt = r'fs_id":([^"]+),'
+    uk_patt = r'uk":([^"]+),'
+    errno_patt = r'errno":([^"]+),'
+    primary_id_patt = r'shareid":([^"]+),'
+    sign = match1(html, sign_patt)
+    timestamp = match1(html, timestamp_patt)
+    appid = match1(html, appid_patt)
+    bdstoken = match1(html, bdstoken_patt)
+    fs_id = match1(html, fs_id_patt)
+    uk = match1(html, uk_patt)
+    primary_id = match1(html, primary_id_patt)
+    return sign, timestamp, bdstoken, appid, primary_id, fs_id, uk
+
+
+def baidu_pan_gen_cookies(url, post_data=None):
+    from http import cookiejar
+    cookiejar = cookiejar.CookieJar()
+    opener = request.build_opener(request.HTTPCookieProcessor(cookiejar))
+    resp = opener.open('http://pan.baidu.com')
+    if post_data != None:
+        resp = opener.open(url, bytes(parse.urlencode(post_data), 'utf-8'))
+    return cookjar2hdr(cookiejar)
+
+
+def baidu_pan_protected_share(url):
+    print('This share is protected by password!')
+    inpwd = input('Please provide unlock password: ')
+    inpwd = inpwd.replace(' ', '').replace('\t', '')
+    print('Please wait...')
+    post_pwd = {
+        'pwd': inpwd,
+        'vcode': None,
+        'vstr': None
+    }
+    from http import cookiejar
+    import time
+    cookiejar = cookiejar.CookieJar()
+    opener = request.build_opener(request.HTTPCookieProcessor(cookiejar))
+    resp = opener.open('http://pan.baidu.com')
+    resp = opener.open(url)
+    init_url = resp.geturl()
+    verify_url = 'http://pan.baidu.com/share/verify?%s&t=%s&channel=chunlei&clienttype=0&web=1' % (
+        init_url.split('?', 1)[1], int(time.time()))
+    refer_url = init_url
+    fake_headers = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'Host': 'pan.baidu.com',
+        'Origin': 'http://pan.baidu.com',
+        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2500.0 Safari/537.36',
+        'Referer': refer_url
+    }
+    opener.addheaders = dict2triplet(fake_headers)
+    pwd_resp = opener.open(verify_url, bytes(
+        parse.urlencode(post_pwd), 'utf-8'))
+    pwd_resp_str = ungzip(pwd_resp.read()).decode('utf-8')
+    pwd_res = json.loads(pwd_resp_str)
+    if pwd_res['errno'] != 0:
+        raise AssertionError(
+            'Server returned an error: %s (Incorrect password?)' % pwd_res['errno'])
+    pg_resp = opener.open('http://pan.baidu.com/share/link?%s' %
+                          init_url.split('?', 1)[1])
+    content = ungzip(pg_resp.read()).decode('utf-8')
+    sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
+        content)
+    psk = query_cookiejar(cookiejar, 'BDCLND')
+    psk = parse.unquote(psk)
+    fake_headers['Cookie'] = cookjar2hdr(cookiejar)
+    return sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk
+
+
+def cookjar2hdr(cookiejar):
+    cookie_str = ''
+    for i in cookiejar:
+        cookie_str = cookie_str + i.name + '=' + i.value + ';'
+    return cookie_str[:-1]
+
+
+def query_cookiejar(cookiejar, name):
+    for i in cookiejar:
+        if i.name == name:
+            return i.value
+
+
+def dict2triplet(dictin):
+    out_triplet = []
+    for i in dictin:
+        out_triplet.append((i, dictin[i]))
+    return out_triplet
+
 site_info = "Baidu.com"
 download = baidu_download
 download_playlist = playlist_not_supported("baidu")

From 4bbafeb9e48e76b7b622f2133685905b362a9096 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Thu, 20 Oct 2016 13:09:30 -0600
Subject: [PATCH 0241/1225] icourse: add supprt

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/icourses.py | 129 +++++++++++++++++++++++++++++
 3 files changed, 131 insertions(+)
 create mode 100644 src/you_get/extractors/icourses.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 948b0ca26e..ca86767398 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -29,6 +29,7 @@
     'huaban'           : 'huaban',
     'huomao'           : 'huomaotv',
     'iask'             : 'sina',
+    'icourses'         : 'icourses',
     'ifeng'            : 'ifeng',
     'imgur'            : 'imgur',
     'in'               : 'alive',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index e69bc2fd01..61b6a0d180 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -24,6 +24,7 @@
 from .google import *
 from .heavymusic import *
 from .huaban import *
+from .icourses import *
 from .ifeng import *
 from .imgur import *
 from .infoq import *
diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
new file mode 100644
index 0000000000..5f9b8edffe
--- /dev/null
+++ b/src/you_get/extractors/icourses.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+from ..common import *
+from urllib import parse
+import xml.etree.ElementTree as ET
+import datetime
+import hashlib
+import base64
+import logging
+from urllib import error
+import re
+
+__all__ = ['icourses_download']
+
+
+def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
+    title, real_url = icourses_cn_url_parser(
+        url, info_only=info_only, **kwargs)
+    if real_url is not None:
+        for tries in range(0, 3):
+            try:
+                _, type_, size = url_info(real_url, faker=True)
+                break
+            except error.HTTPError:
+                logging.warning('Failed to fetch the video file! Retrying...')
+                title, real_url = icourses_cn_url_parser(url)
+        print_info(site_info, title, type_, size)
+        if not info_only:
+            download_urls([real_url], title, 'flv',
+                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
+
+
+def icourses_playlist_download(url, **kwargs):
+    import random
+    from time import sleep
+    html = get_content(url)
+    page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
+    video_js_number = r'changeforvideo\((.*?)\)'
+    fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
+    page_navi_vars = re.search(pattern=page_type_patt, string=html)
+    dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
+        page_navi_vars.group(2), page_navi_vars.group(1))
+    html = get_content(dummy_page)
+    fs_status = match1(html, fs_flag)
+    video_list = re.findall(pattern=video_js_number, string=html)
+    for video in video_list:
+        video_args = video.replace('\'', '').split(',')
+        video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
+            video_args[0], video_args[1], fs_status or '1')
+        sleep(random.Random().randint(0, 5))  # Prevent from blockage
+        icourses_download(url=video_url, **kwargs)
+
+
+def icourses_cn_url_parser(url, **kwargs):
+    PLAYER_BASE_VER = '150606-1'
+    ENCRYPT_MOD_VER = '151020'
+    ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
+    html = get_content(url)
+    if re.search(pattern=r'showSectionNode\(.*\)', string=html):
+        logging.warning('Switching to playlist mode!')
+        return icourses_playlist_download(url, **kwargs)
+    flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
+    server_time_patt = r'MPlayer.swf\?v\=(\d+)'
+    uuid_patt = r'uuid:(\d+)'
+    other_args_patt = r'other:"(.*)"'
+    res_url_patt = r'IService:\'([^\']+)'
+    title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
+    title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
+    title_a = match1(html, title_a_patt).strip()
+    title_b = match1(html, title_b_patt).strip()
+    title = title_a + title_b  # WIP, FIXME
+    title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
+                   unescape_html(title).replace(' ', ''))
+    server_time = match1(html, server_time_patt)
+    flashvars = match1(html, flashvars_patt)
+    uuid = match1(flashvars, uuid_patt)
+    other_args = match1(flashvars, other_args_patt)
+    res_url = match1(flashvars, res_url_patt)
+    url_parts = {'v': server_time, 'other': other_args,
+                 'uuid': uuid, 'IService': res_url}
+    req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
+    logging.debug('Requesting video resource location...')
+    xml_resp = get_html(req_url)
+    xml_obj = ET.fromstring(xml_resp)
+    logging.debug('The result was {}'.format(xml_obj.get('status')))
+    if xml_obj.get('status') != 'success':
+        raise ValueError('Server returned error!')
+    common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+                   'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
+                   'start': 0}
+    media_host = xml_obj.find(".//*[@name='host']").text
+    media_url = media_host + xml_obj.find(".//*[@name='url']").text
+    # This is what they called `SSLModule`... But obviously, just a kind of
+    # encryption, takes absolutely no effect in protecting data intergrity
+    if xml_obj.find(".//*[@name='ssl']").text != 'true':
+        logging.debug('The encryption mode is disabled')
+        # when the so-called `SSLMode` is not activated, the parameters, `h`
+        # and `p` can be found in response
+        arg_h = xml_obj.find(".//*[@name='h']").text
+        assert arg_h
+        arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+        url_args = common_args.copy()
+        url_args.update({'h': arg_h, 'r': arg_r})
+        final_url = '{}?{}'.format(
+            media_url, parse.urlencode(url_args))
+        return title, final_url
+    # when the `SSLMode` is activated, we need to receive the timestamp and the
+    # time offset (?) value from the server
+    logging.debug('The encryption mode is in effect')
+    ssl_callback = get_html('{}/ssl/ssl.shtml'.format(media_host)).split(',')
+    ssl_timestamp = int(datetime.datetime.strptime(
+        ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
+    sign_this = ENCRYPT_SALT + \
+        parse.urlparse(media_url).path + str(ssl_timestamp)
+    arg_h = base64.b64encode(hashlib.md5(bytes(sign_this, 'utf-8')).digest())
+    # Post-processing, may subject to change, so leaving this alone...
+    arg_h = arg_h.decode('utf-8').strip('=').replace('+',
+                                                     '-').replace('/', '_')
+    arg_r = ssl_timestamp
+    url_args = common_args.copy()
+    url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
+    final_url = '{}?{}'.format(
+        media_url, parse.urlencode(url_args))
+    logging.debug('Concat`ed URL: {}'.format(final_url))
+    return title, final_url
+
+
+site_info = 'icourses.cn'
+download = icourses_download
+download_playlist = icourses_playlist_download

From 5351121186c2c8c94bc7b24419ea5ca305582462 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 12:52:30 -0600
Subject: [PATCH 0242/1225] icouses: Code clean up

---
 src/you_get/extractors/icourses.py | 197 +++++++++++++++--------------
 1 file changed, 105 insertions(+), 92 deletions(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index 5f9b8edffe..5c2f8cda74 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -13,8 +13,9 @@
 
 
 def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
-    title, real_url = icourses_cn_url_parser(
-        url, info_only=info_only, **kwargs)
+    icourses_parser = ICousesExactor(url=url)
+    real_url = icourses_parser.icourses_cn_url_parser(**kwargs)
+    title = icourses_parser.title
     if real_url is not None:
         for tries in range(0, 3):
             try:
@@ -22,108 +23,120 @@ def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
                 break
             except error.HTTPError:
                 logging.warning('Failed to fetch the video file! Retrying...')
-                title, real_url = icourses_cn_url_parser(url)
+                real_url = icourses_parser.icourses_cn_url_parser()
+                title = icourses_parser.title
         print_info(site_info, title, type_, size)
         if not info_only:
             download_urls([real_url], title, 'flv',
                           total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
 
 
-def icourses_playlist_download(url, **kwargs):
-    import random
-    from time import sleep
-    html = get_content(url)
-    page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
-    video_js_number = r'changeforvideo\((.*?)\)'
-    fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
-    page_navi_vars = re.search(pattern=page_type_patt, string=html)
-    dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
-        page_navi_vars.group(2), page_navi_vars.group(1))
-    html = get_content(dummy_page)
-    fs_status = match1(html, fs_flag)
-    video_list = re.findall(pattern=video_js_number, string=html)
-    for video in video_list:
-        video_args = video.replace('\'', '').split(',')
-        video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
-            video_args[0], video_args[1], fs_status or '1')
-        sleep(random.Random().randint(0, 5))  # Prevent from blockage
-        icourses_download(url=video_url, **kwargs)
+# Why not using VideoExtractor: This site needs specical download method
+class ICousesExactor(object):
 
+    def __init__(self, url):
+        self.url = url
+        self.title = ''
+        return
 
-def icourses_cn_url_parser(url, **kwargs):
-    PLAYER_BASE_VER = '150606-1'
-    ENCRYPT_MOD_VER = '151020'
-    ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
-    html = get_content(url)
-    if re.search(pattern=r'showSectionNode\(.*\)', string=html):
-        logging.warning('Switching to playlist mode!')
-        return icourses_playlist_download(url, **kwargs)
-    flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
-    server_time_patt = r'MPlayer.swf\?v\=(\d+)'
-    uuid_patt = r'uuid:(\d+)'
-    other_args_patt = r'other:"(.*)"'
-    res_url_patt = r'IService:\'([^\']+)'
-    title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
-    title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
-    title_a = match1(html, title_a_patt).strip()
-    title_b = match1(html, title_b_patt).strip()
-    title = title_a + title_b  # WIP, FIXME
-    title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
-                   unescape_html(title).replace(' ', ''))
-    server_time = match1(html, server_time_patt)
-    flashvars = match1(html, flashvars_patt)
-    uuid = match1(flashvars, uuid_patt)
-    other_args = match1(flashvars, other_args_patt)
-    res_url = match1(flashvars, res_url_patt)
-    url_parts = {'v': server_time, 'other': other_args,
-                 'uuid': uuid, 'IService': res_url}
-    req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
-    logging.debug('Requesting video resource location...')
-    xml_resp = get_html(req_url)
-    xml_obj = ET.fromstring(xml_resp)
-    logging.debug('The result was {}'.format(xml_obj.get('status')))
-    if xml_obj.get('status') != 'success':
-        raise ValueError('Server returned error!')
-    common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
-                   'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
-                   'start': 0}
-    media_host = xml_obj.find(".//*[@name='host']").text
-    media_url = media_host + xml_obj.find(".//*[@name='url']").text
-    # This is what they called `SSLModule`... But obviously, just a kind of
-    # encryption, takes absolutely no effect in protecting data intergrity
-    if xml_obj.find(".//*[@name='ssl']").text != 'true':
-        logging.debug('The encryption mode is disabled')
-        # when the so-called `SSLMode` is not activated, the parameters, `h`
-        # and `p` can be found in response
-        arg_h = xml_obj.find(".//*[@name='h']").text
-        assert arg_h
-        arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+    def icourses_playlist_download(self, **kwargs):
+        import random
+        from time import sleep
+        html = get_content(url)
+        page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
+        video_js_number = r'changeforvideo\((.*?)\)'
+        fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
+        page_navi_vars = re.search(pattern=page_type_patt, string=html)
+        dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
+            page_navi_vars.group(2), page_navi_vars.group(1))
+        html = get_content(dummy_page)
+        fs_status = match1(html, fs_flag)
+        video_list = re.findall(pattern=video_js_number, string=html)
+        for video in video_list:
+            video_args = video.replace('\'', '').split(',')
+            video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
+                video_args[0], video_args[1], fs_status or '1')
+            sleep(random.Random().randint(0, 5))  # Prevent from blockage
+            icourses_download(video_url, **kwargs)
+
+    def icourses_cn_url_parser(self, **kwargs):
+        PLAYER_BASE_VER = '150606-1'
+        ENCRYPT_MOD_VER = '151020'
+        ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
+        html = get_content(self.url)
+        if re.search(pattern=r'showSectionNode\(.*\)', string=html):
+            logging.warning('Switching to playlist mode!')
+            return self.icourses_playlist_download(**kwargs)
+        flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
+        server_time_patt = r'MPlayer.swf\?v\=(\d+)'
+        uuid_patt = r'uuid:(\d+)'
+        other_args_patt = r'other:"(.*)"'
+        res_url_patt = r'IService:\'([^\']+)'
+        title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
+        title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
+        title_a = match1(html, title_a_patt).strip()
+        title_b = match1(html, title_b_patt).strip()
+        title = title_a + title_b  # WIP, FIXME
+        title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
+                       unescape_html(title).replace(' ', ''))
+        server_time = match1(html, server_time_patt)
+        flashvars = match1(html, flashvars_patt)
+        uuid = match1(flashvars, uuid_patt)
+        other_args = match1(flashvars, other_args_patt)
+        res_url = match1(flashvars, res_url_patt)
+        url_parts = {'v': server_time, 'other': other_args,
+                     'uuid': uuid, 'IService': res_url}
+        req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
+        logging.debug('Requesting video resource location...')
+        xml_resp = get_html(req_url)
+        xml_obj = ET.fromstring(xml_resp)
+        logging.debug('The result was {}'.format(xml_obj.get('status')))
+        if xml_obj.get('status') != 'success':
+            raise ValueError('Server returned error!')
+        common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+                       'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
+                       'start': 0}
+        media_host = xml_obj.find(".//*[@name='host']").text
+        media_url = media_host + xml_obj.find(".//*[@name='url']").text
+        # This is what they called `SSLModule`... But obviously, just a kind of
+        # encryption, takes absolutely no effect in protecting data intergrity
+        if xml_obj.find(".//*[@name='ssl']").text != 'true':
+            logging.debug('The encryption mode is disabled')
+            # when the so-called `SSLMode` is not activated, the parameters, `h`
+            # and `p` can be found in response
+            arg_h = xml_obj.find(".//*[@name='h']").text
+            assert arg_h
+            arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
+            url_args = common_args.copy()
+            url_args.update({'h': arg_h, 'r': arg_r})
+            final_url = '{}?{}'.format(
+                media_url, parse.urlencode(url_args))
+            self.title = title
+            return final_url
+        # when the `SSLMode` is activated, we need to receive the timestamp and the
+        # time offset (?) value from the server
+        logging.debug('The encryption mode is in effect')
+        ssl_callback = get_html(
+            '{}/ssl/ssl.shtml'.format(media_host)).split(',')
+        ssl_timestamp = int(datetime.datetime.strptime(
+            ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
+        sign_this = ENCRYPT_SALT + \
+            parse.urlparse(media_url).path + str(ssl_timestamp)
+        arg_h = base64.b64encode(hashlib.md5(
+            bytes(sign_this, 'utf-8')).digest())
+        # Post-processing, may subject to change, so leaving this alone...
+        arg_h = arg_h.decode('utf-8').strip('=').replace('+',
+                                                         '-').replace('/', '_')
+        arg_r = ssl_timestamp
         url_args = common_args.copy()
-        url_args.update({'h': arg_h, 'r': arg_r})
+        url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
         final_url = '{}?{}'.format(
             media_url, parse.urlencode(url_args))
-        return title, final_url
-    # when the `SSLMode` is activated, we need to receive the timestamp and the
-    # time offset (?) value from the server
-    logging.debug('The encryption mode is in effect')
-    ssl_callback = get_html('{}/ssl/ssl.shtml'.format(media_host)).split(',')
-    ssl_timestamp = int(datetime.datetime.strptime(
-        ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
-    sign_this = ENCRYPT_SALT + \
-        parse.urlparse(media_url).path + str(ssl_timestamp)
-    arg_h = base64.b64encode(hashlib.md5(bytes(sign_this, 'utf-8')).digest())
-    # Post-processing, may subject to change, so leaving this alone...
-    arg_h = arg_h.decode('utf-8').strip('=').replace('+',
-                                                     '-').replace('/', '_')
-    arg_r = ssl_timestamp
-    url_args = common_args.copy()
-    url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
-    final_url = '{}?{}'.format(
-        media_url, parse.urlencode(url_args))
-    logging.debug('Concat`ed URL: {}'.format(final_url))
-    return title, final_url
+        logging.debug('Crafted URL: {}'.format(final_url))
+        self.title = title
+        return final_url
 
 
 site_info = 'icourses.cn'
 download = icourses_download
-download_playlist = icourses_playlist_download
+# download_playlist = icourses_playlist_download

From ae4e533ec9d28fb1598fb91dfa87ce16cb06bc92 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 14:03:21 -0600
Subject: [PATCH 0243/1225] common: add dynamic url support for
 `url_save_chunked`

---
 src/you_get/common.py | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 948b0ca26e..0f7fd0e390 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -547,7 +547,11 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
         os.remove(filepath) # on Windows rename could fail if destination filepath exists
     os.rename(temp_filepath, filepath)
 
-def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}):
+def url_save_chunked(url, filepath, bar, dyn_callback=None, chunk_size=0, ignore_range=False, refer=None, is_part=False, faker=False, headers={}):
+    def dyn_update_url(received):
+        if callable(dyn_callback):
+            logging.debug('Calling callback %s for new URL from %s' % (dyn_callback.__name__, received))
+            return dyn_callback(received)
     if os.path.exists(filepath):
         if not force:
             if not is_part:
@@ -585,19 +589,26 @@ def url_save_chunked(url, filepath, bar, refer = None, is_part = False, faker =
     else:
         headers = {}
     if received:
-        headers['Range'] = 'bytes=' + str(received) + '-'
+        url = dyn_update_url(received)
+        if not ignore_range:
+            headers['Range'] = 'bytes=' + str(received) + '-'
     if refer:
         headers['Referer'] = refer
 
-    response = request.urlopen(request.Request(url, headers = headers), None)
+    response = request.urlopen(request.Request(url, headers=headers), None)
 
     with open(temp_filepath, open_mode) as output:
+        this_chunk = received
         while True:
             buffer = response.read(1024 * 256)
             if not buffer:
                 break
             output.write(buffer)
             received += len(buffer)
+            if chunk_size and (received - this_chunk) >= chunk_size:
+                url = dyn_callback(received)
+                this_chunk = received
+                response = request.urlopen(request.Request(url, headers=headers), None)
             if bar:
                 bar.update_received(len(buffer))
 
@@ -846,7 +857,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
 
     print()
 
-def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}):
+def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
     assert urls
     if dry_run:
         print('Real URLs:\n%s\n' % urls)
@@ -860,7 +871,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
 
     filename = '%s.%s' % (title, ext)
     filepath = os.path.join(output_dir, filename)
-    if total_size and ext in ('ts'):
+    if total_size:
         if not force and os.path.exists(filepath[:-3] + '.mkv'):
             print('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
             print()
@@ -875,7 +886,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
         print('Downloading %s ...' % tr(filename))
         filepath = os.path.join(output_dir, filename)
         parts.append(filepath)
-        url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers)
+        url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
         bar.done()
 
         if not merge:

From 2183448c9098c1abd0e9cf47fa305e3775e1e098 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Tue, 25 Oct 2016 14:15:23 -0600
Subject: [PATCH 0244/1225] icourses: implement fake `keep connection alive`

---
 src/you_get/extractors/icourses.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index 5c2f8cda74..cb2ff74aee 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 from ..common import *
 from urllib import parse
+import random
+from time import sleep
 import xml.etree.ElementTree as ET
 import datetime
 import hashlib
@@ -12,23 +14,24 @@
 __all__ = ['icourses_download']
 
 
-def icourses_download(url, info_only, merge=False, output_dir='.', **kwargs):
+def icourses_download(url, merge=False, output_dir='.', **kwargs):
     icourses_parser = ICousesExactor(url=url)
     real_url = icourses_parser.icourses_cn_url_parser(**kwargs)
     title = icourses_parser.title
     if real_url is not None:
-        for tries in range(0, 3):
+        for tries in range(0, 5):
             try:
                 _, type_, size = url_info(real_url, faker=True)
                 break
             except error.HTTPError:
                 logging.warning('Failed to fetch the video file! Retrying...')
+                sleep(random.Random().randint(0, 5))  # Prevent from blockage
                 real_url = icourses_parser.icourses_cn_url_parser()
                 title = icourses_parser.title
         print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([real_url], title, 'flv',
-                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True)
+        if not kwargs['info_only']:
+            download_urls_chunked([real_url], title, 'flv',
+                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True, ignore_range=True, chunk_size=15000000, dyn_callback=icourses_parser.icourses_cn_url_parser)
 
 
 # Why not using VideoExtractor: This site needs specical download method
@@ -40,9 +43,7 @@ def __init__(self, url):
         return
 
     def icourses_playlist_download(self, **kwargs):
-        import random
-        from time import sleep
-        html = get_content(url)
+        html = get_content(self.url)
         page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
         video_js_number = r'changeforvideo\((.*?)\)'
         fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
@@ -59,7 +60,7 @@ def icourses_playlist_download(self, **kwargs):
             sleep(random.Random().randint(0, 5))  # Prevent from blockage
             icourses_download(video_url, **kwargs)
 
-    def icourses_cn_url_parser(self, **kwargs):
+    def icourses_cn_url_parser(self, received=0, **kwargs):
         PLAYER_BASE_VER = '150606-1'
         ENCRYPT_MOD_VER = '151020'
         ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
@@ -93,9 +94,14 @@ def icourses_cn_url_parser(self, **kwargs):
         logging.debug('The result was {}'.format(xml_obj.get('status')))
         if xml_obj.get('status') != 'success':
             raise ValueError('Server returned error!')
-        common_args = {'lv': PLAYER_BASE_VER, 'ls': 'play',
+        if received:
+            play_type = 'seek'
+        else:
+            play_type = 'play'
+            received -= 1
+        common_args = {'lv': PLAYER_BASE_VER, 'ls': play_type,
                        'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
-                       'start': 0}
+                       'start': received + 1}
         media_host = xml_obj.find(".//*[@name='host']").text
         media_url = media_host + xml_obj.find(".//*[@name='url']").text
         # This is what they called `SSLModule`... But obviously, just a kind of

From ac33461c88344d86f74b69572f2f27d03fd708b5 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 27 Oct 2016 17:44:02 +0800
Subject: [PATCH 0245/1225] fix(huomao): adapt to new url format

---
 src/you_get/extractors/huomaotv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/huomaotv.py b/src/you_get/extractors/huomaotv.py
index 4852ff06f3..6e98c80054 100644
--- a/src/you_get/extractors/huomaotv.py
+++ b/src/you_get/extractors/huomaotv.py
@@ -6,7 +6,7 @@
 
 
 def get_mobile_room_url(room_id):
-    return 'http://www.huomao.com/mobile/mob_live?cid=%s' % room_id
+    return 'http://www.huomao.com/mobile/mob_live/%s' % room_id
 
 
 def get_m3u8_url(stream_id):

From 0f3fe97e9caedf976286193aff5dddf430d80962 Mon Sep 17 00:00:00 2001
From: Cheng Gu <guchengf@gmail.com>
Date: Thu, 27 Oct 2016 17:44:54 +0800
Subject: [PATCH 0246/1225] update: add huomao.com

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index b994ebd10b..182fc12a6d 100644
--- a/README.md
+++ b/README.md
@@ -407,6 +407,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
+| 火猫TV   | <http://www.huomao.com/>         |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 

From feffcb656ad2c33b17fb2e20598f8137fc69789c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 00:24:31 +0200
Subject: [PATCH 0247/1225] [processor.ffmpeg] fix params in
 ffmpeg_download_stream

---
 src/you_get/processor/ffmpeg.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1c0ba1a3de..c6da97f78d 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -212,15 +212,6 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     if not (output_dir == '.'):
         output = output_dir + '/' + output
 
-    ffmpeg_params = []
-    #should these exist...
-    if params is not None:
-        if len(params) > 0:
-            for k, v in params:
-                ffmpeg_params.append(k)
-                ffmpeg_params.append(v)
-
-
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
     ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
@@ -230,6 +221,12 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
     else:
         ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 
+    if params is not None:
+        if len(params) > 0:
+            for k, v in params:
+                ffmpeg_params.append(k)
+                ffmpeg_params.append(v)
+
     ffmpeg_params.append(output)
 
     print(' '.join(ffmpeg_params))

From 4b55884e86df68c56ae9fce85293f9b757e97576 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 00:26:25 +0200
Subject: [PATCH 0248/1225] [dailymotion] use ffmpeg_download_stream, fix #1466

---
 src/you_get/extractors/dailymotion.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index 8b701cd17d..2e96c160ed 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -4,6 +4,11 @@
 
 from ..common import *
 
+def extract_m3u(url):
+    content = get_content(url)
+    m3u_url = re.findall(r'http://.*', content)[0]
+    return match1(m3u_url, r'([^#]+)')
+
 def dailymotion_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     """Downloads Dailymotion videos by URL.
     """
@@ -13,7 +18,7 @@ def dailymotion_download(url, output_dir = '.', merge = True, info_only = False,
     title = match1(html, r'"video_title"\s*:\s*"([^"]+)"') or \
             match1(html, r'"title"\s*:\s*"([^"]+)"')
 
-    for quality in ['720','480','380','240','auto']:
+    for quality in ['1080','720','480','380','240','auto']:
         try:
             real_url = info[quality][0]["url"]
             if real_url:
@@ -21,11 +26,12 @@ def dailymotion_download(url, output_dir = '.', merge = True, info_only = False,
         except KeyError:
             pass
 
-    type, ext, size = url_info(real_url)
+    m3u_url = extract_m3u(real_url)
+    mime, ext, size = 'video/mp4', 'mp4', 0
 
-    print_info(site_info, title, type, size)
+    print_info(site_info, title, mime, size)
     if not info_only:
-        download_urls([real_url], title, ext, size, output_dir, merge = merge)
+        download_url_ffmpeg(m3u_url, title, ext, output_dir=output_dir, merge=merge)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From a4f4fb362616862cc283b05122e74be346f1a309 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 30 Oct 2016 16:16:04 +0100
Subject: [PATCH 0249/1225] Revert "fix for #1405" (fix #1485)

This reverts commit 38ba0dbe48ecac4b7a354e4cf5766cf9415fb3c9.
---
 src/you_get/extractors/youku.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 1fb09e8cbf..853a75ba68 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -314,9 +314,6 @@ def extract(self, **kwargs):
                                 q         = q
                             )
                         ksegs += [i['server'] for i in json.loads(get_content(u))]
-
-                        if (parse_host(ksegs[len(ksegs)-1])[0] == "vali.cp31.ott.cibntv.net"):
-                            ksegs.pop(len(ksegs)-1)
             except error.HTTPError as e:
                 # Use fallback stream data in case of HTTP 404
                 log.e('[Error] ' + str(e))

From e8514d1370bc748946940c7c2f757db5c9cf42c8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 01:44:04 +0100
Subject: [PATCH 0250/1225] version 0.4.575

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 6d91656c9d..6d4f6c4fc3 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.555'
+__version__ = '0.4.575'

From 391ca5643a355c310db786e467c6929fd5dde53f Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Wed, 2 Nov 2016 20:44:40 -0400
Subject: [PATCH 0251/1225] [embed] correct tudou pattern

Hyphen-minus (-) is a valid character in Tudou's video ID. It's even
present in the second pattern of tudou_embed_patterns, just not the
first.
---
 src/you_get/extractors/embed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index a177e66394..fc4015c4ee 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -25,7 +25,7 @@
 """
 http://www.tudou.com/programs/view/html5embed.action?type=0&amp;code=3LS_URGvl54&amp;lcode=&amp;resourceId=0_06_05_99
 """
-tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_]+)\&',
+tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_-]+)\&',
                          'www\.tudou\.com/v/([a-zA-Z0-9_-]+)/[^"]*v\.swf'
                        ]
 

From 2b0fe3443f844690305caa0a468d1b744c72ced5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 17:03:01 +0100
Subject: [PATCH 0252/1225] [test] remove test_vimeo

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 638206af77..0fa2979a31 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -21,9 +21,6 @@ def test_magisto(self):
     def test_mixcloud(self):
         mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)
 
-    def test_vimeo(self):
-        vimeo.download("http://vimeo.com/56810854", info_only=True)
-
     def test_youtube(self):
         youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
         youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)

From bc590cbd62ca4350598551e41910c719864f0c36 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 21:32:13 +0100
Subject: [PATCH 0253/1225] [douban] add support: movie.douban.com

---
 README.md                        |  4 ++--
 src/you_get/extractors/douban.py | 23 +++++++++++++++++------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 182fc12a6d..40a26803db 100644
--- a/README.md
+++ b/README.md
@@ -128,7 +128,7 @@ $ you-get https://github.com/soimort/you-get/archive/master.zip
 or use [chocolatey package manager](https://chocolatey.org):
 
 ```
-> choco upgrade you-get 
+> choco upgrade you-get
 ```
 
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
@@ -373,7 +373,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 爆米花网 | <http://www.baomihua.com/>     |✓| | |
 | **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
 | Dilidili | <http://www.dilidili.com/>     |✓| | |
-| 豆瓣     | <http://www.douban.com/>       | | |✓|
+| 豆瓣     | <http://www.douban.com/>       |✓| |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
 | Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
diff --git a/src/you_get/extractors/douban.py b/src/you_get/extractors/douban.py
index 187e99c031..1a4a67d179 100644
--- a/src/you_get/extractors/douban.py
+++ b/src/you_get/extractors/douban.py
@@ -7,12 +7,23 @@
 
 def douban_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_html(url)
-    if 'subject' in url:
+
+    if re.match(r'https?://movie', url):
+        title = match1(html, 'name="description" content="([^"]+)')
+        tid = match1(url, 'trailer/(\d+)')
+        real_url = 'https://movie.douban.com/trailer/video_url?tid=%s' % tid
+        type, ext, size = url_info(real_url)
+
+        print_info(site_info, title, type, size)
+        if not info_only:
+            download_urls([real_url], title, ext, size, output_dir, merge = merge)
+
+    elif 'subject' in url:
         titles = re.findall(r'data-title="([^"]*)">', html)
         song_id = re.findall(r'<li class="song-item" id="([^"]*)"', html)
         song_ssid = re.findall(r'data-ssid="([^"]*)"', html)
         get_song_url = 'http://music.douban.com/j/songlist/get_song_url'
-        
+
         for i in range(len(titles)):
             title = titles[i]
             datas = {
@@ -35,16 +46,16 @@ def douban_download(url, output_dir = '.', merge = True, info_only = False, **kw
                 except:
                     pass
 
-    else: 
+    else:
         titles = re.findall(r'"name":"([^"]*)"', html)
         real_urls = [re.sub('\\\\/', '/', i) for i in re.findall(r'"rawUrl":"([^"]*)"', html)]
-        
+
         for i in range(len(titles)):
             title = titles[i]
             real_url = real_urls[i]
-            
+
             type, ext, size = url_info(real_url)
-            
+
             print_info(site_info, title, type, size)
             if not info_only:
                 download_urls([real_url], title, ext, size, output_dir, merge = merge)

From 5601e1fe30316d02ff6f51b4d77689d004ba2f13 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 3 Nov 2016 22:03:56 +0100
Subject: [PATCH 0254/1225] [bilibili] fix support for bangumi

---
 src/you_get/extractors/bilibili.py | 108 +++++++++++++++--------------
 1 file changed, 56 insertions(+), 52 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c18290b8be..122dea0bb9 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -119,66 +119,70 @@ def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_o
 def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_content(url)
 
-    if re.match(r'https?://bangumi\.bilibili\.com/', url):
-        # quick hack for bangumi URLs
-        url = r1(r'"([^"]+)" class="v-av-link"', html)
-        html = get_content(url)
-
     title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
                    r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
     if title:
         title = unescape_html(title)
         title = escape_file_path(title)
 
-    flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
-                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-    assert flashvars
-    flashvars = flashvars.replace(': ', '=')
-    t, cid = flashvars.split('=', 1)
-    cid = cid.split('&')[0]
-    if t == 'cid':
-        if re.match(r'https?://live\.bilibili\.com/', url):
-            title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
-            bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+    if re.match(r'https?://bangumi\.bilibili\.com/', url):
+        # quick hack for bangumi URLs
+        episode_id = r1(r'data-current-episode-id="(\d+)"', html)
+        cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
+                            post_data={'episode_id': episode_id})
+        cid = json.loads(cont)['result']['cid']
+        bilibili_download_by_cid(str(cid), title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-        else:
-            # multi-P
-            cids = []
-            pages = re.findall('<option value=\'([^\']*)\'', html)
-            titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-            for i, page in enumerate(pages):
-                html = get_html("http://www.bilibili.com%s" % page)
-                flashvars = r1_of([r'(cid=\d+)',
-                                   r'flashvars="([^"]+)"',
-                                   r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-                if flashvars:
-                    t, cid = flashvars.split('=', 1)
-                    cids.append(cid.split('&')[0])
-                if url.endswith(page):
-                    cids = [cid.split('&')[0]]
-                    titles = [titles[i]]
-                    break
-
-            # no multi-P
-            if not pages:
-                cids = [cid]
-                titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-
-            for i in range(len(cids)):
-                bilibili_download_by_cid(cids[i],
-                                         titles[i],
-                                         output_dir=output_dir,
-                                         merge=merge,
-                                         info_only=info_only)
-
-    elif t == 'vid':
-        sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'ykid':
-        youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif t == 'uid':
-        tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
-        raise NotImplementedError(flashvars)
+        flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
+                           r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+        assert flashvars
+        flashvars = flashvars.replace(': ', '=')
+        t, cid = flashvars.split('=', 1)
+        cid = cid.split('&')[0]
+        if t == 'cid':
+            if re.match(r'https?://live\.bilibili\.com/', url):
+                title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
+                bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+
+            else:
+                # multi-P
+                cids = []
+                pages = re.findall('<option value=\'([^\']*)\'', html)
+                titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
+                for i, page in enumerate(pages):
+                    html = get_html("http://www.bilibili.com%s" % page)
+                    flashvars = r1_of([r'(cid=\d+)',
+                                       r'flashvars="([^"]+)"',
+                                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
+                    if flashvars:
+                        t, cid = flashvars.split('=', 1)
+                        cids.append(cid.split('&')[0])
+                    if url.endswith(page):
+                        cids = [cid.split('&')[0]]
+                        titles = [titles[i]]
+                        break
+
+                # no multi-P
+                if not pages:
+                    cids = [cid]
+                    titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
+
+                for i in range(len(cids)):
+                    bilibili_download_by_cid(cids[i],
+                                             titles[i],
+                                             output_dir=output_dir,
+                                             merge=merge,
+                                             info_only=info_only)
+
+        elif t == 'vid':
+            sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        elif t == 'ykid':
+            youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        elif t == 'uid':
+            tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
+        else:
+            raise NotImplementedError(flashvars)
 
     if not info_only and not dry_run:
         if not kwargs['caption']:

From d04997ec9bc2ce68655334063e5cce840053a0b0 Mon Sep 17 00:00:00 2001
From: Rokic <Rokic.github@gmail.com>
Date: Tue, 8 Nov 2016 02:09:39 +0800
Subject: [PATCH 0255/1225] fix #1415

Songs from netease cloud music playlist will have a prefix indicates
their order in the list.
---
 src/you_get/extractors/netease.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 63ee59b8a8..d5f3b1fa9f 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -54,13 +54,15 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
                 os.mkdir(new_dir)
             cover_url = j['result']['coverImgUrl']
             download_urls([cover_url], "cover", "jpg", 0, new_dir)
-
-        for i in j['result']['tracks']:
-            netease_song_download(i, output_dir=new_dir, info_only=info_only)
+        
+        prefix_width = len(str(len(j['result']['tracks'])))
+        for n, i in enumerate(j['result']['tracks']):
+            playlist_prefix = '%%.%dd_' % prefix_width % n
+            netease_song_download(i, output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
             try: # download lyrics
                 assert kwargs['caption']
                 l = loads(get_content("http://music.163.com/api/song/lyric/?id=%s&lv=-1&csrf_token=" % i['id'], headers={"Referer": "http://music.163.com/"}))
-                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only)
+                netease_lyric_download(i, l["lrc"]["lyric"], output_dir=new_dir, info_only=info_only, playlist_prefix=playlist_prefix)
             except: pass
 
     elif "song" in url:
@@ -85,10 +87,10 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
         j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
         netease_video_download(j['data'], output_dir=output_dir, info_only=info_only)
 
-def netease_lyric_download(song, lyric, output_dir='.', info_only=False):
+def netease_lyric_download(song, lyric, output_dir='.', info_only=False, playlist_prefix=""):
     if info_only: return
 
-    title = "%s. %s" % (song['position'], song['name'])
+    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
     filename = '%s.lrc' % get_filename(title)
     print('Saving %s ...' % filename, end="", flush=True)
     with open(os.path.join(output_dir, filename),
@@ -103,8 +105,8 @@ def netease_video_download(vinfo, output_dir='.', info_only=False):
     netease_download_common(title, url_best,
                             output_dir=output_dir, info_only=info_only)
 
-def netease_song_download(song, output_dir='.', info_only=False):
-    title = "%s. %s" % (song['position'], song['name'])
+def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix=""):
+    title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
     songNet = 'p' + song['mp3Url'].split('/')[2][1:]
 
     if 'hMusic' in song and song['hMusic'] != None:

From 51dd7ad8e6b757687a4c06af7b6b3fb3dfa5f5b1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 9 Nov 2016 17:13:02 +0100
Subject: [PATCH 0256/1225] [youtube] use url_encoded_fmt_stream_map from video
 page, fix #1502

---
 src/you_get/extractors/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 33e3923e67..64af5c1470 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -155,6 +155,8 @@ def prepare(self, **kwargs):
                 try:
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
                     self.html5player = 'https:' + ytplayer_config['assets']['js']
+                    # Workaround: get_video_info returns bad s. Why?
+                    stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 except:
                     self.html5player = None
 

From 78ba20266c6b5e1fef1398af60ea8361bf57fff0 Mon Sep 17 00:00:00 2001
From: moyo <moyo@uuland.org>
Date: Sun, 13 Nov 2016 17:41:00 +0800
Subject: [PATCH 0257/1225] 1. Change container from FLV to TS 2. Fix video url
 matcher 3. Use m3u8 ext-info for fast size calculate 4. Use m3u8 url for
 video playing

---
 src/you_get/extractors/mgtv.py | 74 ++++++++++++++++++++++++++--------
 1 file changed, 57 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index aeb4249072..3ce62efebc 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -12,11 +12,11 @@
 class MGTV(VideoExtractor):
     name = "芒果 (MGTV)"
 
-    # Last updated: 2015-11-24
+    # Last updated: 2016-11-13
     stream_types = [
-        {'id': 'hd', 'container': 'flv', 'video_profile': '超清'},
-        {'id': 'sd', 'container': 'flv', 'video_profile': '高清'},
-        {'id': 'ld', 'container': 'flv', 'video_profile': '标清'},
+        {'id': 'hd', 'container': 'ts', 'video_profile': '超清'},
+        {'id': 'sd', 'container': 'ts', 'video_profile': '高清'},
+        {'id': 'ld', 'container': 'ts', 'video_profile': '标清'},
     ]
     
     id_dic = {i['video_profile']:(i['id']) for i in stream_types}
@@ -27,7 +27,7 @@ class MGTV(VideoExtractor):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        return match1(url, 'http://www.mgtv.com/v/\d/\d+/\w+/(\d+).html')
+        return match1(url, 'http://www.mgtv.com/b/\d+/(\d+).html')
     
     #----------------------------------------------------------------------
     @staticmethod
@@ -44,10 +44,15 @@ def get_mgtv_real_url(url):
 
         content = get_content(content['info'])  #get the REAL M3U url, maybe to be changed later?
         segment_list = []
+        segments_size = 0
         for i in content.split():
             if not i.startswith('#'):  #not the best way, better we use the m3u8 package
                 segment_list.append(base_url + i)
-        return segment_list
+            # use ext-info for fast size calculate
+            elif i.startswith('#EXT-MGTV-File-SIZE:'):
+                segments_size += int(i[i.rfind(':')+1:])
+
+        return m3u_url, segments_size, segment_list
 
     def download_playlist_by_url(self, url, **kwargs):
         pass
@@ -69,28 +74,25 @@ def prepare(self, **kwargs):
                 quality_id = self.id_dic[s['video_profile']]
                 url = stream_available[s['video_profile']]
                 url = re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
-                segment_list_this = self.get_mgtv_real_url(url)
-                
-                container_this_stream = ''
-                size_this_stream = 0
+                m3u8_url, m3u8_size, segment_list_this = self.get_mgtv_real_url(url)
+
                 stream_fileid_list = []
                 for i in segment_list_this:
-                    _, container_this_stream, size_this_seg = url_info(i)
-                    size_this_stream += size_this_seg
                     stream_fileid_list.append(os.path.basename(i).split('.')[0])
-                    
+
             #make pieces
             pieces = []
             for i in zip(stream_fileid_list, segment_list_this):
                 pieces.append({'fileid': i[0], 'segs': i[1],})
 
                 self.streams[quality_id] = {
-                        'container': 'flv',
+                        'container': s['container'],
                         'video_profile': s['video_profile'],
-                        'size': size_this_stream,
-                        'pieces': pieces
+                        'size': m3u8_size,
+                        'pieces': pieces,
+                        'm3u8_url': m3u8_url
                     }
-                
+
             if not kwargs['info_only']:
                 self.streams[quality_id]['src'] = segment_list_this
 
@@ -107,6 +109,44 @@ def extract(self, **kwargs):
             # Extract stream with the best quality
             stream_id = self.streams_sorted[0]['id']
 
+    def download(self, **kwargs):
+
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            stream_id = kwargs['stream_id']
+        else:
+            stream_id = 'null'
+
+        # print video info only
+        if 'info_only' in kwargs and kwargs['info_only']:
+            if stream_id != 'null':
+                if 'index' not in kwargs:
+                    self.p(stream_id)
+                else:
+                    self.p_i(stream_id)
+            else:
+                # Display all available streams
+                if 'index' not in kwargs:
+                    self.p([])
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    self.p_i(stream_id)
+
+        # default to use the best quality
+        if stream_id == 'null':
+            stream_id = self.streams_sorted[0]['id']
+
+        stream_info = self.streams[stream_id]
+
+        if not kwargs['info_only']:
+            if player:
+                # with m3u8 format because some video player can process urls automatically (e.g. mpv)
+                launch_player(player, [stream_info['m3u8_url']])
+            else:
+                download_urls(stream_info['src'], self.title, stream_info['container'], stream_info['size'],
+                              output_dir=kwargs['output_dir'],
+                              merge=kwargs['merge'],
+                              av=stream_id in self.dash_streams)
+
 site = MGTV()
 download = site.download_by_url
 download_playlist = site.download_playlist_by_url
\ No newline at end of file

From 65713cae2cf1c122be72c2d6fdaf854b35260562 Mon Sep 17 00:00:00 2001
From: L <z2d@jifangcheng.com>
Date: Mon, 14 Nov 2016 21:49:13 +0800
Subject: [PATCH 0258/1225] update yixia_download url match rule

resolved #1346
---
 src/you_get/extractors/yixia.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index ca5c4bd6ab..7d5ba29089 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -51,11 +51,11 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         yixia_download_by_scid = yixia_miaopai_download_by_scid
         site_info = "Yixia Miaopai"
         
-        if re.match(r'http://www.miaopai.com/show/channel/\w+', url):  #PC
+        if re.match(r'http://www.miaopai.com/show/channel/.+', url):  #PC
             scid = match1(url, r'http://www.miaopai.com/show/channel/(.+)\.htm')
-        elif re.match(r'http://www.miaopai.com/show/\w+', url):  #PC
+        elif re.match(r'http://www.miaopai.com/show/.+', url):  #PC
             scid = match1(url, r'http://www.miaopai.com/show/(.+)\.htm')
-        elif re.match(r'http://m.miaopai.com/show/channel/\w+', url):  #Mobile
+        elif re.match(r'http://m.miaopai.com/show/channel/.+', url):  #Mobile
             scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)\.htm')
     
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu

From a7635e96a5e20cc4025fbcb236254e7a69c6556c Mon Sep 17 00:00:00 2001
From: Zhang Cheng <stephenpcg@gmail.com>
Date: Thu, 17 Nov 2016 11:18:01 +0800
Subject: [PATCH 0259/1225] [mgtv] add bsf:a aac_adtstoasc to ffmpeg args, fix
 #1458.

---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1c0ba1a3de..dcc8e1c86d 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -125,7 +125,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
 
         params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
         params.append(output + '.txt')
-        params += ['-c', 'copy', output]
+        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', output]
 
         subprocess.check_call(params)
         os.remove(output + '.txt')

From 250672f42d475eba1b7a69b48683cf0d0576698a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 19 Nov 2016 20:47:18 +0100
Subject: [PATCH 0260/1225] version 0.4.595

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 6d4f6c4fc3..2891990646 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.575'
+__version__ = '0.4.595'

From fe2720544a804926a24aba33f44c98d3706ec3bd Mon Sep 17 00:00:00 2001
From: Yohohaha <390342156@qq.com>
Date: Fri, 25 Nov 2016 20:03:38 +0800
Subject: [PATCH 0261/1225] fix syntax error

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9faaa93963..0100cae785 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -969,7 +969,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
     assert has_ffmpeg_installed(), "FFmpeg not installed."
     global output_filename
-    if(output_filename)
+    if(output_filename):
         dotPos = output_filename.rfind(".")
         title = output_filename[:dotPos]
         ext = output_filename[dotPos+1:]

From e65c2d23a0cdfe622c15a740f1c04384c7813563 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Nov 2016 13:07:21 +0100
Subject: [PATCH 0262/1225] [tudou] fix #1526

---
 src/you_get/extractors/tudou.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index 6bbbc12bb5..8c434437c7 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -32,11 +32,11 @@ def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only =
 def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if 'acfun.tudou.com' in url:  #wrong way!
         url = url.replace('acfun.tudou.com', 'www.acfun.tv')
-        you_get.extractors.acfun.acfun_download(url, output_dir, 
-                                               merge, 
+        you_get.extractors.acfun.acfun_download(url, output_dir,
+                                               merge,
                                                info_only)
         return  #throw you back
-    
+
     # Embedded player
     id = r1(r'http://www.tudou.com/v/([^/]+)/', url)
     if id:
@@ -44,7 +44,7 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwa
 
     html = get_decoded_html(url)
 
-    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
+    title = r1(r'\Wkw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
     assert title
     title = unescape_html(title)
 

From 03266c030a254dac2103a3c2a2d086e36fb9dc9a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Nov 2016 12:35:50 +0100
Subject: [PATCH 0263/1225] [youtube] fix dash-mpd for live streams (no
 yt:contentLength field)

---
 src/you_get/extractors/youtube.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 64af5c1470..61dc2cb732 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -258,11 +258,17 @@ def prepare(self, **kwargs):
                     burls = rep.getElementsByTagName('BaseURL')
                     dash_mp4_a_url = burls[0].firstChild.nodeValue
                     dash_mp4_a_size = burls[0].getAttribute('yt:contentLength')
+                    if not dash_mp4_a_size:
+                        try: dash_mp4_a_size = url_size(dash_mp4_a_url)
+                        except: continue
                 elif mimeType == 'audio/webm':
                     rep = aset.getElementsByTagName('Representation')[-1]
                     burls = rep.getElementsByTagName('BaseURL')
                     dash_webm_a_url = burls[0].firstChild.nodeValue
                     dash_webm_a_size = burls[0].getAttribute('yt:contentLength')
+                    if not dash_webm_a_size:
+                        try: dash_webm_a_size = url_size(dash_webm_a_url)
+                        except: continue
                 elif mimeType == 'video/mp4':
                     for rep in aset.getElementsByTagName('Representation'):
                         w = int(rep.getAttribute('width'))
@@ -271,6 +277,9 @@ def prepare(self, **kwargs):
                         burls = rep.getElementsByTagName('BaseURL')
                         dash_url = burls[0].firstChild.nodeValue
                         dash_size = burls[0].getAttribute('yt:contentLength')
+                        if not dash_size:
+                            try: dash_size = url_size(dash_url)
+                            except: continue
                         self.dash_streams[itag] = {
                             'quality': '%sx%s' % (w, h),
                             'itag': itag,
@@ -288,6 +297,9 @@ def prepare(self, **kwargs):
                         burls = rep.getElementsByTagName('BaseURL')
                         dash_url = burls[0].firstChild.nodeValue
                         dash_size = burls[0].getAttribute('yt:contentLength')
+                        if not dash_size:
+                            try: dash_size = url_size(dash_url)
+                            except: continue
                         self.dash_streams[itag] = {
                             'quality': '%sx%s' % (w, h),
                             'itag': itag,

From 538f1796f203297ef9e66c0a9d07691daa28df97 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Nov 2016 17:09:28 +0100
Subject: [PATCH 0264/1225] [universal] workaround for websites that block HEAD
 requests

---
 src/you_get/common.py               | 6 +++---
 src/you_get/extractors/universal.py | 5 ++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 0100cae785..27998cf5c6 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -338,7 +338,7 @@ def get_content(url, headers={}, decoded=True):
         if charset is not None:
             data = data.decode(charset)
         else:
-            data = data.decode('utf-8')
+            data = data.decode('utf-8', 'ignore')
 
     return data
 
@@ -395,12 +395,12 @@ def url_size(url, faker = False, headers = {}):
 def urls_size(urls, faker = False, headers = {}):
     return sum([url_size(url, faker=faker, headers=headers) for url in urls])
 
-def get_head(url, headers = {}):
+def get_head(url, headers = {}, get_method = 'HEAD'):
     if headers:
         req = request.Request(url, headers = headers)
     else:
         req = request.Request(url)
-    req.get_method = lambda : 'HEAD'
+    req.get_method = lambda : get_method
     res = request.urlopen(req)
     return dict(res.headers)
 
diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index ebab70f88f..a4262f618d 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -6,7 +6,10 @@
 from .embed import *
 
 def universal_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    content_type = get_head(url, headers=fake_headers)['Content-Type']
+    try:
+        content_type = get_head(url, headers=fake_headers)['Content-Type']
+    except:
+        content_type = get_head(url, headers=fake_headers, get_method='GET')['Content-Type']
     if content_type.startswith('text/html'):
         try:
             embed_download(url, output_dir, merge=merge, info_only=info_only)

From 8e150e69897724d315c3e31cbc187511a0d2d54c Mon Sep 17 00:00:00 2001
From: sheerluck <sheer.luck.andrew@gmail.com>
Date: Mon, 28 Nov 2016 18:01:42 +0300
Subject: [PATCH 0265/1225] fix for NameError: name 'output_json' is not
 defined

---
 src/you_get/extractors/qq.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index f170752713..c9ee7c0f1b 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -56,12 +56,12 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
             if not info_only:
                 download_urls(part_urls, parts_ti, ext, total_size, output_dir=output_dir, merge=merge)
         else:
-            fvkey = output_json['vl']['vi'][0]['fvkey']
-            mp4 = output_json['vl']['vi'][0]['cl'].get('ci', None)
+            fvkey = video_json['vl']['vi'][0]['fvkey']
+            mp4 = video_json['vl']['vi'][0]['cl'].get('ci', None)
             if mp4:
                 mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
             else:
-                mp4 = output_json['vl']['vi'][0]['fn']
+                mp4 = video_json['vl']['vi'][0]['fn']
             url = '%s/%s?vkey=%s' % ( parts_prefix, mp4, fvkey )
             _, ext, size = url_info(url, faker=True)
 

From 474f4d724a796426db99c398dfe56756549cd223 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Dec 2016 17:40:29 +0100
Subject: [PATCH 0266/1225] [common] pass valid filename in download_url_ffmpeg

---
 src/you_get/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 27998cf5c6..7db4fba2b1 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -968,11 +968,15 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
 
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
     assert has_ffmpeg_installed(), "FFmpeg not installed."
+
     global output_filename
-    if(output_filename):
+    if output_filename:
         dotPos = output_filename.rfind(".")
         title = output_filename[:dotPos]
         ext = output_filename[dotPos+1:]
+
+    title = tr(get_filename(title))
+
     ffmpeg_download_stream(url, title, ext, params, output_dir)
 
 def playlist_not_supported(name):

From 61d9bf124edf5bd89283eb5e373cabae5e8953b6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Dec 2016 17:41:23 +0100
Subject: [PATCH 0267/1225] [youtube] download hlsvp via ffmpeg

---
 src/you_get/extractors/youtube.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 61dc2cb732..c403cb7400 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -148,6 +148,17 @@ def prepare(self, **kwargs):
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
                 self.title = parse.unquote_plus(video_info['title'][0])
+
+                # YouTube Live
+                if 'url_encoded_fmt_stream_map' not in video_info:
+                    hlsvp = video_info['hlsvp'][0]
+
+                    if 'info_only' in kwargs and kwargs['info_only']:
+                        return
+                    else:
+                        download_url_ffmpeg(hlsvp, self.title, 'mp4')
+                        exit(0)
+
                 stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
 
                 # Parse video page (for DASH)

From 606e0a786e2ab631288d2f4567ed1d37334ae52e Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Sun, 4 Dec 2016 19:36:17 -0500
Subject: [PATCH 0268/1225] [lizhi] overhaul

Lizhi extractor has stopped working. In particular, there are two major
changes:

- URL format change: no more #/ in URL paths;
- The /api/audio/{radio_id}/{audio_id} API now returns 404.

This is a rewrite based on the /api/radio_audios API.
---
 src/you_get/extractors/lizhi.py | 74 ++++++++++++++++++++-------------
 1 file changed, 46 insertions(+), 28 deletions(-)

diff --git a/src/you_get/extractors/lizhi.py b/src/you_get/extractors/lizhi.py
index 56dbf7561b..65988a9f0c 100644
--- a/src/you_get/extractors/lizhi.py
+++ b/src/you_get/extractors/lizhi.py
@@ -4,37 +4,55 @@
 import json
 from ..common import *
 
-def lizhi_download_playlist(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    # like this http://www.lizhi.fm/#/31365/
-    #api desc: s->start l->length band->some radio
-    #http://www.lizhi.fm/api/radio_audios?s=0&l=100&band=31365
-    band_id = match1(url,r'#/(\d+)')
-    #try to get a considerable large l to reduce html parsing task.
-    api_url = 'http://www.lizhi.fm/api/radio_audios?s=0&l=65535&band='+band_id
-    content_json = json.loads(get_content(api_url))
-    for sound in content_json:
-        title = sound["name"]
-        res_url = sound["url"]
-        songtype, ext, size = url_info(res_url,faker=True)
-        print_info(site_info, title, songtype, size)
-        if not info_only:
-            #no referer no speed!
-            download_urls([res_url], title, ext, size, output_dir, merge=merge ,refer = 'http://www.lizhi.fm',faker=True)    
-    pass
+# radio_id: e.g. 549759 from http://www.lizhi.fm/549759/
+#
+# Returns a list of tuples (audio_id, title, url) for each episode
+# (audio) in the radio playlist. url is the direct link to the audio
+# file.
+def lizhi_extract_playlist_info(radio_id):
+    # /api/radio_audios API parameters:
+    #
+    # - s: starting episode
+    # - l: count (per page)
+    # - band: radio_id
+    #
+    # We use l=65535 for poor man's pagination (that is, no pagination
+    # at all -- hope all fits on a single page).
+    #
+    # TODO: Use /api/radio?band={radio_id} to get number of episodes
+    # (au_cnt), then handle pagination properly.
+    api_url = 'http://www.lizhi.fm/api/radio_audios?s=0&l=65535&band=%s' % radio_id
+    api_response = json.loads(get_content(api_url))
+    return [(ep['id'], ep['name'], ep['url']) for ep in api_response]
 
-def lizhi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    # url like http://www.lizhi.fm/#/549759/18864883431656710
-    api_id = match1(url,r'#/(\d+/\d+)')
-    api_url = 'http://www.lizhi.fm/api/audio/'+api_id
-    content_json = json.loads(get_content(api_url))
-    title = content_json["audio"]["name"]
-    res_url = content_json["audio"]["url"]
-    songtype, ext, size = url_info(res_url,faker=True)
-    print_info(site_info, title, songtype, size)
+def lizhi_download_audio(audio_id, title, url, output_dir='.', info_only=False):
+    filetype, ext, size = url_info(url)
+    print_info(site_info, title, filetype, size)
     if not info_only:
-        #no referer no speed!
-        download_urls([res_url], title, ext, size, output_dir, merge=merge ,refer = 'http://www.lizhi.fm',faker=True)    
+        download_urls([url], title, ext, size, output_dir=output_dir)
 
+def lizhi_download_playlist(url, output_dir='.', info_only=False, **kwargs):
+    # Sample URL: http://www.lizhi.fm/549759/
+    radio_id = match1(url,r'/(\d+)')
+    if not radio_id:
+        raise NotImplementedError('%s not supported' % url)
+    for audio_id, title, url in lizhi_extract_playlist_info(radio_id):
+        lizhi_download_audio(audio_id, title, url, output_dir=output_dir, info_only=info_only)
+
+def lizhi_download(url, output_dir='.', info_only=False, **kwargs):
+    # Sample URL: http://www.lizhi.fm/549759/18864883431656710/
+    m = re.search(r'/(?P<radio_id>\d+)/(?P<audio_id>\d+)', url)
+    if not m:
+        raise NotImplementedError('%s not supported' % url)
+    radio_id = m.group('radio_id')
+    audio_id = m.group('audio_id')
+    # Look for the audio_id among the full list of episodes
+    for aid, title, url in lizhi_extract_playlist_info(radio_id):
+        if aid == audio_id:
+            lizhi_download_audio(audio_id, title, url, output_dir=output_dir, info_only=info_only)
+            break
+    else:
+        raise NotImplementedError('Audio #%s not found in playlist #%s' % (audio_id, radio_id))
 
 site_info = "lizhi.fm"
 download = lizhi_download

From a6d3c13684cff5811e3c1c6bac93698355cc3a43 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Mon, 5 Dec 2016 23:45:28 -0500
Subject: [PATCH 0269/1225] [embed] add support for bilibili's embedded player

Sample embed: for http://www.bilibili.com/video/av5079467/:

  <embed
    height="415" width="544" quality="high"
    allowfullscreen="true" type="application/x-shockwave-flash"
    src="http://static.hdslb.com/miniloader.swf"
    flashvars="aid=5079467&page=1"
    pluginspage="http://www.adobe.com/shockwave/download/download.cgi?P1_Prod_Version=ShockwaveFlash"
  ></embed>
---
 src/you_get/extractors/embed.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index fc4015c4ee..3bdb924ca1 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -2,6 +2,7 @@
 
 from ..common import *
 
+from .bilibili import bilibili_download
 from .iqiyi import iqiyi_download_by_vid
 from .le import letvcloud_download_by_vu
 from .netease import netease_download
@@ -42,6 +43,11 @@
 
 vimeo_embed_patters = [ 'player\.vimeo\.com/video/(\d+)' ]
 
+"""
+check the share button on http://www.bilibili.com/video/av5079467/
+"""
+bilibili_embed_patterns = [ 'static\.hdslb\.com/miniloader\.swf.*aid=(\d+)' ]
+
 
 def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
     content = get_content(url, headers=fake_headers)
@@ -78,6 +84,12 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
+    aids = matchall(content, bilibili_embed_patterns)
+    for aid in aids:
+        found = True
+        url = 'http://www.bilibili.com/video/av%s/' % aid
+        bilibili_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
+
     if not found:
         raise NotImplementedError(url)
 

From 44e60c3e2193d3198899f211a8b7c9767b0b6d5e Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Sat, 10 Dec 2016 12:23:35 +0100
Subject: [PATCH 0270/1225] Initial support for yizhibo.com

---
 src/you_get/common.py             |  1 +
 src/you_get/extractors/yizhibo.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 src/you_get/extractors/yizhibo.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7db4fba2b1..fd727cf49a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -91,6 +91,7 @@
     'xiaojiadianvideo' : 'fc2video',
     'yinyuetai'        : 'yinyuetai',
     'miaopai'          : 'yixia',
+    'yizhibo'          : 'yizhibo',
     'youku'            : 'youku',
     'youtu'            : 'youtube',
     'youtube'          : 'youtube',
diff --git a/src/you_get/extractors/yizhibo.py b/src/you_get/extractors/yizhibo.py
new file mode 100644
index 0000000000..f524a0a8b8
--- /dev/null
+++ b/src/you_get/extractors/yizhibo.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+__all__ = ['yizhibo_download']
+
+from ..common import *
+import json
+import time
+
+def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    video_id = url[url.rfind('/')+1:].split(".")[0]
+    json_request_url = 'http://www.yizhibo.com/live/h5api/get_basic_live_info?scid={}'.format(video_id)
+    content = get_html(json_request_url)
+    error = json.loads(content)['result']
+    if (error != 1):
+        raise ValueError("Error : {}".format(error))
+
+    data = json.loads(content)#['data']
+    title = data.get('data')['live_title']
+    if (title == ''):
+        title = data.get('data')['nickname']
+    real_url = data.get('data')['play_url']
+
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
+
+site_info = "yizhibo.com"
+download = yizhibo_download
+download_playlist = playlist_not_supported('yizhibo')

From 0f33e471ad65c2c2dfb0a1e4480cb39d1f2430a2 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Sat, 10 Dec 2016 12:26:06 +0100
Subject: [PATCH 0271/1225] minor correction

---
 src/you_get/extractors/yizhibo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/yizhibo.py b/src/you_get/extractors/yizhibo.py
index f524a0a8b8..0744e1f91d 100644
--- a/src/you_get/extractors/yizhibo.py
+++ b/src/you_get/extractors/yizhibo.py
@@ -14,7 +14,7 @@ def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **k
     if (error != 1):
         raise ValueError("Error : {}".format(error))
 
-    data = json.loads(content)#['data']
+    data = json.loads(content)
     title = data.get('data')['live_title']
     if (title == ''):
         title = data.get('data')['nickname']

From 0f1d5beb1494ca6b64b90e3d8d5949de29b2c31b Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Sun, 11 Dec 2016 01:46:23 +0100
Subject: [PATCH 0272/1225] Changed the plugin to use download_urls instead of
 ffmpeg

---
 src/you_get/extractors/yizhibo.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/yizhibo.py b/src/you_get/extractors/yizhibo.py
index 0744e1f91d..37fa043c6b 100644
--- a/src/you_get/extractors/yizhibo.py
+++ b/src/you_get/extractors/yizhibo.py
@@ -9,7 +9,7 @@
 def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     video_id = url[url.rfind('/')+1:].split(".")[0]
     json_request_url = 'http://www.yizhibo.com/live/h5api/get_basic_live_info?scid={}'.format(video_id)
-    content = get_html(json_request_url)
+    content = get_content(json_request_url)
     error = json.loads(content)['result']
     if (error != 1):
         raise ValueError("Error : {}".format(error))
@@ -18,11 +18,17 @@ def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **k
     title = data.get('data')['live_title']
     if (title == ''):
         title = data.get('data')['nickname']
-    real_url = data.get('data')['play_url']
-
-    print_info(site_info, title, 'flv', float('inf'))
+    m3u8_url = data.get('data')['play_url']
+    m3u8 = get_content(m3u8_url)
+    base_url = "/".join(data.get('data')['play_url'].split("/")[:7])+"/"
+    part_url = re.findall(r'([0-9]+\.ts)', m3u8)
+    real_url = []
+    for i in part_url:
+        url = base_url + i
+        real_url.append(url)
+    print_info(site_info, title, 'ts', float('inf'))
     if not info_only:
-        download_url_ffmpeg(real_url, title, 'flv', None, output_dir, merge = merge)
+        download_urls(real_url, title, 'ts', float('inf'), output_dir, merge = merge)
 
 site_info = "yizhibo.com"
 download = yizhibo_download

From e0554b2d7b7a214c988100ac32187208b22e1d26 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Sun, 11 Dec 2016 01:49:13 +0100
Subject: [PATCH 0273/1225] Made player use the m3u8 file.

---
 src/you_get/extractors/yizhibo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/yizhibo.py b/src/you_get/extractors/yizhibo.py
index 37fa043c6b..11ce86ad3b 100644
--- a/src/you_get/extractors/yizhibo.py
+++ b/src/you_get/extractors/yizhibo.py
@@ -28,6 +28,8 @@ def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **k
         real_url.append(url)
     print_info(site_info, title, 'ts', float('inf'))
     if not info_only:
+        if player:
+            launch_player(player, [m3u8_url])
         download_urls(real_url, title, 'ts', float('inf'), output_dir, merge = merge)
 
 site_info = "yizhibo.com"

From 9905620b5297483e5e10195aad90a14be1d360fd Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Fri, 16 Dec 2016 09:36:29 +0100
Subject: [PATCH 0274/1225] Fix for magisto

---
 src/you_get/extractors/magisto.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/magisto.py b/src/you_get/extractors/magisto.py
index 2a53be02ad..b2e8e50258 100644
--- a/src/you_get/extractors/magisto.py
+++ b/src/you_get/extractors/magisto.py
@@ -3,15 +3,19 @@
 __all__ = ['magisto_download']
 
 from ..common import *
+import json
 
 def magisto_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
-
-    title1 = r1(r'<meta name="twitter:title" content="([^"]*)"', html)
-    title2 = r1(r'<meta name="twitter:description" content="([^"]*)"', html)
-    video_hash = r1(r'http://www.magisto.com/video/([^/]+)', url)
-    title = "%s %s - %s" % (title1, title2, video_hash)
-    url = r1(r'<source type="[^"]+" src="([^"]*)"', html)
+    
+    video_hash = r1(r'video\/([a-zA-Z0-9]+)', url)
+    api_url = 'https://www.magisto.com/api/video/{}'.format(video_hash)
+    content = get_html(api_url)
+    data = json.loads(content)
+    title1 = data['title']
+    title2 = data['creator']
+    title = "%s - %s" % (title1, title2)
+    url = data['video_direct_url']
     type, ext, size = url_info(url)
 
     print_info(site_info, title, type, size)

From db16bcb659ae98cfdbf69fa8c4acbf913fd6ab7f Mon Sep 17 00:00:00 2001
From: Zhang <Naville@users.noreply.github.com>
Date: Thu, 22 Dec 2016 22:33:37 +0800
Subject: [PATCH 0275/1225] [BiliBili] Better Multi-Part Video Naming

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 122dea0bb9..043c3753bc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -167,10 +167,10 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                 if not pages:
                     cids = [cid]
                     titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-
                 for i in range(len(cids)):
+                    completeTitle=title+"-"+titles[i]#Build Better Title
                     bilibili_download_by_cid(cids[i],
-                                             titles[i],
+                                             completeTitle,
                                              output_dir=output_dir,
                                              merge=merge,
                                              info_only=info_only)

From bc5ff346d043e8097b81902d6f5392fc3e7869fc Mon Sep 17 00:00:00 2001
From: Zhang <Naville@users.noreply.github.com>
Date: Fri, 23 Dec 2016 11:50:51 +0800
Subject: [PATCH 0276/1225] [BiliBili] revert naming for single part videos

---
 src/you_get/extractors/bilibili.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 043c3753bc..2e54ed47fa 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -168,7 +168,11 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
                     cids = [cid]
                     titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
                 for i in range(len(cids)):
-                    completeTitle=title+"-"+titles[i]#Build Better Title
+                    completeTitle=None
+                    if (title == titles[i]):
+                        completeTitle=title
+                    else:
+                        completeTitle=title+"-"+titles[i]#Build Better Title
                     bilibili_download_by_cid(cids[i],
                                              completeTitle,
                                              output_dir=output_dir,

From af4db738a2f2e9e23ef192145a0ece286f1a4c67 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 24 Dec 2016 15:49:47 +0100
Subject: [PATCH 0277/1225] [test] remove mixcloud

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 0fa2979a31..020455b044 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -18,9 +18,6 @@ def test_imgur(self):
     def test_magisto(self):
         magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)
 
-    def test_mixcloud(self):
-        mixcloud.download("http://www.mixcloud.com/DJVadim/north-america-are-you-ready/", info_only=True)
-
     def test_youtube(self):
         youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
         youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)

From b493af9a69878544ddc6a1fdb71ca61b48bd57ab Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Thu, 15 Dec 2016 23:37:35 -0500
Subject: [PATCH 0278/1225] [ffmpeg] fix concat list when output dir is not pwd

Relative paths in the concat list are considered relative to the parent
directory of the script, not the calling directory. This isn't entirely
obvious from the documentation, but it is easy to infer from the concat
demuxer's concept of "safety", and easy to test (confirmed on FFmpeg
3.2.2). See https://ffmpeg.org/ffmpeg-all.html#concat-1 for details.

This commit fixes the wrong relative paths when --output-dir is
specified and not pwd.

This commit also
- Factors out common concat list writer code;
- Slightly simplifies the code to collect FFmpeg params (on Py35+ we can
  further simplify by unpacking LOGLEVEL with the star operator right in
  the list literal).
---
 src/you_get/processor/ffmpeg.py | 56 ++++++++++++++-------------------
 1 file changed, 23 insertions(+), 33 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index a8599e527e..433aff3fcc 100644
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -26,6 +26,18 @@ def get_usable_ffmpeg(cmd):
 def has_ffmpeg_installed():
     return FFMPEG is not None
 
+# Given a list of segments and the output path, generates the concat
+# list and returns the path to the concat list.
+def generate_concat_list(files, output):
+    concat_list_path = output + '.txt'
+    concat_list_dir = os.path.dirname(concat_list_path)
+    with open(concat_list_path, 'w', encoding='utf-8') as concat_list:
+        for file in files:
+            if os.path.isfile(file):
+                relpath = os.path.relpath(file, start=concat_list_dir)
+                concat_list.write('file %s\n' % parameterize(relpath))
+    return concat_list_path
+
 def ffmpeg_concat_av(files, output, ext):
     print('Merging video parts... ', end="", flush=True)
     params = [FFMPEG] + LOGLEVEL
@@ -52,17 +64,9 @@ def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
 def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w', encoding="utf-8")
-        for file in files:
-            if os.path.isfile(file):
-                concat_list.write("file %s\n" % parameterize(file))
-        concat_list.close()
-
-        params = [FFMPEG] + LOGLEVEL
-        params.extend(['-f', 'concat', '-safe', '-1', '-y', '-i'])
-        params.append(output + '.txt')
-        params += ['-c', 'copy', output]
-
+        concat_list = generate_concat_list(files, output)
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+                                        '-i', concat_list, '-c', 'copy', output]
         if subprocess.call(params) == 0:
             os.remove(output + '.txt')
             return True
@@ -115,18 +119,10 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     print('Merging video parts... ', end="", flush=True)
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w', encoding="utf-8")
-        for file in files:
-            if os.path.isfile(file):
-                # for escaping rules, see:
-                # https://www.ffmpeg.org/ffmpeg-utils.html#Quoting-and-escaping
-                concat_list.write("file %s\n" % parameterize(file))
-        concat_list.close()
-
-        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
-        params.append(output + '.txt')
-        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', output]
-
+        concat_list = generate_concat_list(files, output)
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+                                        '-i', concat_list, '-c', 'copy',
+                                        '-bsf:a', 'aac_adtstoasc', output]
         subprocess.check_call(params)
         os.remove(output + '.txt')
         return True
@@ -162,16 +158,10 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     print('Merging video parts... ', end="", flush=True)
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
-        concat_list = open(output + '.txt', 'w', encoding="utf-8")
-        for file in files:
-            if os.path.isfile(file):
-                concat_list.write("file %s\n" % parameterize(file))
-        concat_list.close()
-
-        params = [FFMPEG] + LOGLEVEL + ['-f', 'concat', '-safe', '-1', '-y', '-i']
-        params.append(output + '.txt')
-        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc', output]
-
+        concat_list = generate_concat_list(files, output)
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+                                        '-i', concat_list, '-c', 'copy',
+                                        '-bsf:a', 'aac_adtstoasc', output]
         subprocess.check_call(params)
         os.remove(output + '.txt')
         return True

From f7b6f6b40f97813206252f9c41dbe05bda592918 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Sun, 25 Dec 2016 13:48:00 -0500
Subject: [PATCH 0279/1225] ffmpeg: set loglevel to info in debug mode

Occasionally, the FFmpeg invocation fails (which could be due to bugs in
you-get; see #1558 for instance), but -loglevel quiet means nothing is
printed other than the exit status (pretty much always 1) in Python's
traceback, which is not helpful at all.

This commit restores FFmpeg's regular output (-loglevel info) when
--debug is specified. We're not using verbose, debug or trace because
those levels are mostly only useful for debugging FFmpeg itself, which
is not our goal.

Due to lack of meaningful API to access the global logging level, this
is a hack based on two assumptions:

1. When --debug is enabled, the root logger level is set to DEBUG;
2. processor.ffmpeg is lazily imported, after command line options are
   parsed.
---
 src/you_get/processor/ffmpeg.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
 mode change 100644 => 100755 src/you_get/processor/ffmpeg.py

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
old mode 100644
new mode 100755
index a8599e527e..f5b3cd38c0
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import logging
 import os.path
 import subprocess
 from ..util.strings import parameterize
@@ -21,7 +22,10 @@ def get_usable_ffmpeg(cmd):
         return None
 
 FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)
-LOGLEVEL = ['-loglevel', 'quiet']
+if logging.getLogger().isEnabledFor(logging.DEBUG):
+    LOGLEVEL = ['-loglevel', 'info']
+else:
+    LOGLEVEL = ['-loglevel', 'quiet']
 
 def has_ffmpeg_installed():
     return FFMPEG is not None

From 927a1cb91f854cb5260f67b15d9811f763955407 Mon Sep 17 00:00:00 2001
From: liujianshan <liujianshan@aipai.com>
Date: Thu, 29 Dec 2016 19:47:53 +0800
Subject: [PATCH 0280/1225] Fix soku.com vid download error problem

---
 src/you_get/extractor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 594b908eef..332440dd07 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -206,7 +206,7 @@ def download(self, **kwargs):
                           output_dir=kwargs['output_dir'],
                           merge=kwargs['merge'],
                           av=stream_id in self.dash_streams)
-            if not kwargs['caption']:
+            if 'caption' not in kwargs or not kwargs['caption']:
                 print('Skipping captions.')
                 return
             for lang in self.caption_tracks:

From 76399e8561c421ead7a590ef857a98eccb16af61 Mon Sep 17 00:00:00 2001
From: ChenYuan <github@zju.email>
Date: Sun, 1 Jan 2017 00:44:56 +0800
Subject: [PATCH 0281/1225] fix bilibili bangumi

modify the regex to get eposide id
---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 122dea0bb9..aecb072cba 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -127,7 +127,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
 
     if re.match(r'https?://bangumi\.bilibili\.com/', url):
         # quick hack for bangumi URLs
-        episode_id = r1(r'data-current-episode-id="(\d+)"', html)
+        episode_id = r1(r'first_ep_id = "(\d+)"', html)
         cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
                             post_data={'episode_id': episode_id})
         cid = json.loads(cont)['result']['cid']

From 60b6834e547e328b1dee86dc748689292beba0e8 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Tue, 3 Jan 2017 23:58:56 +0100
Subject: [PATCH 0282/1225] Quanmin support.

---
 README.md                         |  1 +
 src/you_get/common.py             |  1 +
 src/you_get/extractors/quanmin.py | 25 +++++++++++++++++++++++++
 3 files changed, 27 insertions(+)
 create mode 100644 src/you_get/extractors/quanmin.py

diff --git a/README.md b/README.md
index 40a26803db..98c403c31b 100644
--- a/README.md
+++ b/README.md
@@ -408,6 +408,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
 | 火猫TV   | <http://www.huomao.com/>         |✓| | |
+| 全民Tv   | <http://www.quanmin.tv/>         |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7db4fba2b1..f320f6ab84 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -65,6 +65,7 @@
     'pptv'             : 'pptv',
     'qianmo'           : 'qianmo',
     'qq'               : 'qq',
+    'quanmin'          : 'quanmin',
     'showroom-live'    : 'showroom',
     'sina'             : 'sina',
     'smgbb'            : 'bilibili',
diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
new file mode 100644
index 0000000000..99e8790c95
--- /dev/null
+++ b/src/you_get/extractors/quanmin.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+__all__ = ['quanmin_download']
+
+from ..common import *
+import json
+import time
+
+def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    roomid = url[url.rfind("/")+1:]
+    json_request_url = 'http://www.quanmin.tv/json/rooms/{}/info4.json'.format(roomid)
+
+    content = get_html(json_request_url)
+    data = json.loads(content)
+
+    title = data["title"]
+    real_url = "http://flv.quanmin.tv/live/{}.flv".format(roomid)
+
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
+
+site_info = "quanmin.tv"
+download = quanmin_download
+download_playlist = playlist_not_supported('quanmin')

From fc2c77effaae54970e40246a1ceded8bcced6dc5 Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Wed, 4 Jan 2017 13:56:32 +0100
Subject: [PATCH 0283/1225] Fixes quanmin, when stream is offline.

---
 src/you_get/extractors/quanmin.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
index 99e8790c95..89d63ea90f 100644
--- a/src/you_get/extractors/quanmin.py
+++ b/src/you_get/extractors/quanmin.py
@@ -9,11 +9,13 @@
 def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     roomid = url[url.rfind("/")+1:]
     json_request_url = 'http://www.quanmin.tv/json/rooms/{}/info4.json'.format(roomid)
-
     content = get_html(json_request_url)
     data = json.loads(content)
 
     title = data["title"]
+    
+    if not data["play_status"]:
+        raise ValueError("The live stream is not online!")
     real_url = "http://flv.quanmin.tv/live/{}.flv".format(roomid)
 
     print_info(site_info, title, 'flv', float('inf'))

From f452eec729ac961c35043a11007f4fd1bfb79c20 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sun, 8 Jan 2017 21:36:03 +0800
Subject: [PATCH 0284/1225] [qq] support for videos embedded in weixin

example url:
http://mp.weixin.qq.com/s?__biz=MzA3OTgxODI4NQ==&mid=2653200488&idx=1&sn=bd6d0279b2430cc208d9da74226871db&chksm=847dbb2ab30a323c4b1735887158daf1e295abe586aff0a646ce4257a48010f80bcfb1379c95&scene=0#rd
---
 src/you_get/extractors/qq.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index c9ee7c0f1b..f2c3d9ece1 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -73,7 +73,14 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
     if 'live.qq.com' in url:
-        qieDownload(url,output_dir=output_dir, merge=merge, info_only=info_only)
+        qieDownload(url, output_dir=output_dir, merge=merge, info_only=info_only)
+        return
+
+    if 'mp.weixin.qq.com/s?' in url:
+        content = get_html(url)
+        vids = matchall(content, [r'\bvid=(\w+)'])
+        for vid in vids:
+            qq_download_by_vid(vid, vid, output_dir, merge, info_only)
         return
 
     #do redirect
@@ -101,8 +108,6 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = match1(content, r'"title":"([^"]+)"') if not title else title
         title = vid if not title else title #general fallback
 
-
-
     qq_download_by_vid(vid, title, output_dir, merge, info_only)
 
 site_info = "QQ.com"

From 64dca2182e3a507b516dca7ed0adfc9102904f1f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 9 Jan 2017 01:14:03 +0100
Subject: [PATCH 0285/1225] [youku] do not override existing proxy handler (fix
 #1546, close #1548)

---
 src/you_get/extractors/youku.py | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 853a75ba68..d673e58c7c 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -143,9 +143,9 @@ def prepare(self, **kwargs):
             })
         else:
             proxy_handler = request.ProxyHandler({})
-        opener = request.build_opener(ssl_context, cookie_handler, proxy_handler)
-        opener.addheaders = [('Cookie','__ysuid={}'.format(time.time()))]
-        request.install_opener(opener)
+        for handler in (ssl_context, cookie_handler, proxy_handler):
+            request._opener.add_handler(handler)
+        request._opener.addheaders = [('Cookie','__ysuid={}'.format(time.time()))]
 
         assert self.url or self.vid
 
@@ -162,7 +162,7 @@ def prepare(self, **kwargs):
             api12_url = kwargs['api12_url']  #86
             self.ctype = kwargs['ctype']
             self.title = kwargs['title']
-            
+
         else:
             api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
             api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
@@ -330,36 +330,36 @@ def extract(self, **kwargs):
 
     def open_download_by_vid(self, client_id, vid, **kwargs):
         """self, str, str, **kwargs->None
-        
+
         Arguments:
         client_id:        An ID per client. For now we only know Acfun's
                           such ID.
-        
+
         vid:              An video ID for each video, starts with "C".
-        
+
         kwargs['embsig']: Youku COOP's anti hotlinking.
                           For Acfun, an API call must be done to Acfun's
                           server, or the "playsign" of the content of sign_url
                           shall be empty.
-        
+
         Misc:
         Override the original one with VideoExtractor.
-        
+
         Author:
         Most of the credit are to @ERioK, who gave his POC.
-        
+
         History:
         Jul.28.2016 Youku COOP now have anti hotlinking via embsig. """
         self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
         self.f_code_2 = 'msjv7h2b'
-        
+
         # as in VideoExtractor
         self.url = None
         self.vid = vid
         self.name = "优酷开放平台 (Youku COOP)"
 
         #A little bit of work before self.prepare
-        
+
         #Change as Jul.28.2016 Youku COOP updates its platform to add ant hotlinking
         if kwargs['embsig']:
             sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}&embsig={embsig}".format(client_id = client_id, video_id = vid, embsig = kwargs['embsig'])
@@ -371,9 +371,9 @@ def open_download_by_vid(self, client_id, vid, **kwargs):
         #to be injected and replace ct10 and 12
         api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
         api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
-        
+
         self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
-        
+
         #exact copy from original VideoExtractor
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             unset_proxy()

From 4b782f92be59e92ad38c3b44fe09d2be3e20c582 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Jan 2017 17:25:37 +0100
Subject: [PATCH 0286/1225] [nanagogo] skip erroneous posts

---
 src/you_get/extractors/nanagogo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index 222659f6fb..9cce9e4c04 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -17,6 +17,8 @@ def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     info = json.loads(get_content(api_url))
 
     items = []
+    if info['data']['posts']['post'] is None:
+        return
     for i in info['data']['posts']['post']['body']:
         if 'image' in i:
             image_url = i['image']

From c401c9b9f83050873fddc2c2ac26fc5e79984e35 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Jan 2017 17:31:57 +0100
Subject: [PATCH 0287/1225] [bilibili] fix #1605

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index aecb072cba..920ab77925 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -127,7 +127,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
 
     if re.match(r'https?://bangumi\.bilibili\.com/', url):
         # quick hack for bangumi URLs
-        episode_id = r1(r'first_ep_id = "(\d+)"', html)
+        episode_id = r1(r'#(\d+)$', url) or r1(r'first_ep_id = "(\d+)"', html)
         cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
                             post_data={'episode_id': episode_id})
         cid = json.loads(cont)['result']['cid']

From a7cd3e2c6e5019dbc07d4c974fe0a751095555bf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Jan 2017 17:45:09 +0100
Subject: [PATCH 0288/1225] [bilibili] bangumi titling with episode_id

---
 src/you_get/extractors/bilibili.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 920ab77925..5f00ffe9ee 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -131,6 +131,7 @@ def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs
         cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
                             post_data={'episode_id': episode_id})
         cid = json.loads(cont)['result']['cid']
+        title = '%s [%s]' % (title, episode_id)
         bilibili_download_by_cid(str(cid), title, output_dir=output_dir, merge=merge, info_only=info_only)
 
     else:

From 866876e59ffefef55353c4a6ca819681014ab763 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Jan 2017 17:46:04 +0100
Subject: [PATCH 0289/1225] version 0.4.626

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 2891990646..2e8e4f414b 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.595'
+__version__ = '0.4.626'

From 7eca091d0df30f84520f3b665754828f33be95ae Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Jan 2017 18:45:28 +0100
Subject: [PATCH 0290/1225] tag classifier: Python 3.6

---
 you-get.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/you-get.json b/you-get.json
index 084657d9d9..594742c21a 100644
--- a/you-get.json
+++ b/you-get.json
@@ -24,6 +24,7 @@
     "Programming Language :: Python :: 3.3",
     "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: 3.5",
+    "Programming Language :: Python :: 3.6",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Multimedia",

From 0c1553b97d981a5ab0ffc7605b8c70646423ce3f Mon Sep 17 00:00:00 2001
From: Valdemar Erk <valdemarerk@gmail.com>
Date: Sun, 15 Jan 2017 12:43:34 +0100
Subject: [PATCH 0291/1225] Fixes #1612

---
 src/you_get/extractors/panda.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
index 3f9ceadec1..45249bd280 100644
--- a/src/you_get/extractors/panda.py
+++ b/src/you_get/extractors/panda.py
@@ -8,22 +8,28 @@
 
 def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     roomid = url[url.rfind('/')+1:]
-    json_request_url = 'http://www.panda.tv/api_room?roomid={}&pub_key=&_={}'.format(roomid, int(time.time()))
+    json_request_url ="http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}".format(roomid, int(time.time()))
     content = get_html(json_request_url)
-    errno = json.loads(content)['errno']
-    errmsg = json.loads(content)['errmsg']
+    api_json = json.loads(content)
+    
+    errno = api_json["errno"]
+    errmsg = api_json["errmsg"]
     if errno:
         raise ValueError("Errno : {}, Errmsg : {}".format(errno, errmsg))
-
-    data = json.loads(content)['data']
-    title = data.get('roominfo')['name']
-    room_key = data.get('videoinfo')['room_key']
-    plflag = data.get('videoinfo')['plflag'].split('_')
-    status = data.get('videoinfo')['status']
+    data = api_json["data"]
+    title = data["roominfo"]["name"]
+    room_key = data["videoinfo"]["room_key"]
+    plflag = data["videoinfo"]["plflag"].split("_")
+    status = data["videoinfo"]["status"]
     if status is not "2":
         raise ValueError("The live stream is not online! (status:%s)" % status)
-    real_url = 'http://pl{}.live.panda.tv/live_panda/{}.flv'.format(plflag[1],room_key)
 
+    data2 = json.loads(data["videoinfo"]["plflag_list"])
+    rid = data2["auth"]["rid"]
+    sign = data2["auth"]["sign"]
+    ts = data2["auth"]["time"]
+    real_url = "http://pl{}.live.panda.tv/live_panda/{}.flv?sign={}&ts={}&rid={}".format(plflag[1], room_key, sign, ts, rid)
+    
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
         download_urls([real_url], title, 'flv', None, output_dir, merge = merge)

From 374e1032db23cebb5f8f22a6de5eff4950bd7bf2 Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Mon, 16 Jan 2017 09:56:24 -0800
Subject: [PATCH 0292/1225] [common] also retry if timeout in url_save,
 post_content, etc.

---
 src/you_get/common.py | 51 ++++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f320f6ab84..bea6e62c56 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -298,6 +298,13 @@ def get_location(url):
     # not to do that
     return response.geturl()
 
+def urlopen_with_retry(*args, **kwargs):
+    for i in range(10):
+        try:
+            return request.urlopen(*args, **kwargs)
+        except socket.timeout:
+            logging.debug('request attempt %s timeout' % str(i + 1))
+
 def get_content(url, headers={}, decoded=True):
     """Gets the content of a URL via sending a HTTP GET request.
 
@@ -317,13 +324,7 @@ def get_content(url, headers={}, decoded=True):
         cookies.add_cookie_header(req)
         req.headers.update(req.unredirected_hdrs)
 
-    for i in range(10):
-        try:
-            response = request.urlopen(req)
-            break
-        except socket.timeout:
-            logging.debug('request attempt %s timeout' % str(i + 1))
-
+    response = urlopen_with_retry(req)
     data = response.read()
 
     # Handle HTTP compression for gzip and deflate (zlib)
@@ -362,7 +363,7 @@ def post_content(url, headers={}, post_data={}, decoded=True):
         cookies.add_cookie_header(req)
         req.headers.update(req.unredirected_hdrs)
     post_data_enc = bytes(parse.urlencode(post_data), 'utf-8')
-    response = request.urlopen(req, data = post_data_enc)
+    response = urlopen_with_retry(req, data=post_data_enc)
     data = response.read()
 
     # Handle HTTP compression for gzip and deflate (zlib)
@@ -384,11 +385,11 @@ def post_content(url, headers={}, post_data={}, decoded=True):
 
 def url_size(url, faker = False, headers = {}):
     if faker:
-        response = request.urlopen(request.Request(url, headers = fake_headers), None)
+        response = urlopen_with_retry(request.Request(url, headers=fake_headers))
     elif headers:
-        response = request.urlopen(request.Request(url, headers = headers), None)
+        response = urlopen_with_retry(request.Request(url, headers=headers))
     else:
-        response = request.urlopen(url)
+        response = urlopen_with_retry(url)
 
     size = response.headers['content-length']
     return int(size) if size!=None else float('inf')
@@ -398,20 +399,20 @@ def urls_size(urls, faker = False, headers = {}):
 
 def get_head(url, headers = {}, get_method = 'HEAD'):
     if headers:
-        req = request.Request(url, headers = headers)
+        req = request.Request(url, headers=headers)
     else:
         req = request.Request(url)
-    req.get_method = lambda : get_method
-    res = request.urlopen(req)
+    req.get_method = lambda: get_method
+    res = urlopen_with_retry(req)
     return dict(res.headers)
 
 def url_info(url, faker = False, headers = {}):
     if faker:
-        response = request.urlopen(request.Request(url, headers = fake_headers), None)
+        response = urlopen_with_retry(request.Request(url, headers=fake_headers))
     elif headers:
-        response = request.urlopen(request.Request(url, headers = headers), None)
+        response = urlopen_with_retry(request.Request(url, headers=headers))
     else:
-        response = request.urlopen(request.Request(url))
+        response = urlopen_with_retry(request.Request(url))
 
     headers = response.headers
 
@@ -460,11 +461,11 @@ def url_locations(urls, faker = False, headers = {}):
     locations = []
     for url in urls:
         if faker:
-            response = request.urlopen(request.Request(url, headers = fake_headers), None)
+            response = urlopen_with_retry(request.Request(url, headers=fake_headers))
         elif headers:
-            response = request.urlopen(request.Request(url, headers = headers), None)
+            response = urlopen_with_retry(request.Request(url, headers=headers))
         else:
-            response = request.urlopen(request.Request(url))
+            response = urlopen_with_retry(request.Request(url))
 
         locations.append(response.url)
     return locations
@@ -514,10 +515,10 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
         if refer:
             headers['Referer'] = refer
 
-        response = request.urlopen(request.Request(url, headers = headers), None)
+        response = urlopen_with_retry(request.Request(url, headers=headers))
         try:
             range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
-            end_length = end = int(response.headers['content-range'][6:].split('/')[1])
+            end_length = int(response.headers['content-range'][6:].split('/')[1])
             range_length = end_length - range_start
         except:
             content_length = response.headers['content-length']
@@ -537,7 +538,7 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
                         break
                     else: # Unexpected termination. Retry request
                         headers['Range'] = 'bytes=' + str(received) + '-'
-                        response = request.urlopen(request.Request(url, headers = headers), None)
+                        response = urlopen_with_retry(request.Request(url, headers=headers))
                 output.write(buffer)
                 received += len(buffer)
                 if bar:
@@ -597,7 +598,7 @@ def dyn_update_url(received):
     if refer:
         headers['Referer'] = refer
 
-    response = request.urlopen(request.Request(url, headers=headers), None)
+    response = urlopen_with_retry(request.Request(url, headers=headers))
 
     with open(temp_filepath, open_mode) as output:
         this_chunk = received
@@ -610,7 +611,7 @@ def dyn_update_url(received):
             if chunk_size and (received - this_chunk) >= chunk_size:
                 url = dyn_callback(received)
                 this_chunk = received
-                response = request.urlopen(request.Request(url, headers=headers), None)
+                response = urlopen_with_retry(request.Request(url, headers=headers))
             if bar:
                 bar.update_received(len(buffer))
 

From 0f131e38d4b7fed6cb232aa346df01858547f921 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 16 Jan 2017 23:29:21 +0100
Subject: [PATCH 0293/1225] [facebook] fix #1615

---
 src/you_get/extractors/facebook.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 2a96fcb03b..9eb9fae953 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -11,11 +11,11 @@ def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     title = r1(r'<title id="pageTitle">(.+)</title>', html)
     sd_urls = list(set([
         unicodize(str.replace(i, '\\/', '/'))
-        for i in re.findall(r'"sd_src_no_ratelimit":"([^"]*)"', html)
+        for i in re.findall(r'sd_src_no_ratelimit:"([^"]*)"', html)
     ]))
     hd_urls = list(set([
         unicodize(str.replace(i, '\\/', '/'))
-        for i in re.findall(r'"hd_src_no_ratelimit":"([^"]*)"', html)
+        for i in re.findall(r'hd_src_no_ratelimit:"([^"]*)"', html)
     ]))
     urls = hd_urls if hd_urls else sd_urls
 

From 015871dfa96d480ceed982ecdf45f911ee5b34a8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 16 Jan 2017 23:49:27 +0100
Subject: [PATCH 0294/1225] [acfun] correct active p title, fix #1617

---
 src/you_get/extractors/acfun.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 87e005fb91..6bb0dca471 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -77,6 +77,8 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = unescape_html(title)
     title = escape_file_path(title)
     assert title
+    if match1(url, r'_(\d+)$'): # current P
+        title = title + " " + r1(r'active">([^<]*)', html)
 
     vid = r1('data-vid="(\d+)"', html)
     up = r1('data-name="([^"]+)"', html)

From a520eb051e797b70eddfecaf5c934259c071bf3c Mon Sep 17 00:00:00 2001
From: AlanYang <knift1128@gmail.com>
Date: Thu, 19 Jan 2017 11:15:42 +0800
Subject: [PATCH 0295/1225] fixed mgtv.com 1.17 change api address and stream
 domain

---
 src/you_get/extractors/mgtv.py | 9 +++++----
 src/you_get/json_output.py     | 5 +++++
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 3ce62efebc..1656ac3c60 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -21,7 +21,7 @@ class MGTV(VideoExtractor):
     
     id_dic = {i['video_profile']:(i['id']) for i in stream_types}
     
-    api_endpoint = 'http://v.api.mgtv.com/player/video?video_id={video_id}'
+    api_endpoint = 'http://pcweb.api.mgtv.com/player/video?video_id={video_id}'
 
     @staticmethod
     def get_vid_from_url(url):
@@ -63,6 +63,7 @@ def prepare(self, **kwargs):
         content = get_content(self.api_endpoint.format(video_id = self.vid))
         content = loads(content)
         self.title = content['data']['info']['title']
+        domain = content['data']['stream_domain'][0]
         
         #stream_avalable = [i['name'] for i in content['data']['stream']]
         stream_available = {}
@@ -73,7 +74,7 @@ def prepare(self, **kwargs):
             if s['video_profile'] in stream_available.keys():
                 quality_id = self.id_dic[s['video_profile']]
                 url = stream_available[s['video_profile']]
-                url = re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
+                url = domain + re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
                 m3u8_url, m3u8_size, segment_list_this = self.get_mgtv_real_url(url)
 
                 stream_fileid_list = []
@@ -144,8 +145,8 @@ def download(self, **kwargs):
             else:
                 download_urls(stream_info['src'], self.title, stream_info['container'], stream_info['size'],
                               output_dir=kwargs['output_dir'],
-                              merge=kwargs['merge'],
-                              av=stream_id in self.dash_streams)
+                              merge=kwargs.get('merge', True))
+                              # av=stream_id in self.dash_streams)
 
 site = MGTV()
 download = site.download_by_url
diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 86a42abccb..3e1bac9f82 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -31,6 +31,11 @@ def print_info(site_info=None, title=None, type=None, size=None):
 
 def download_urls(urls=None, title=None, ext=None, total_size=None, refer=None):
     ve = last_info
+    if not ve:
+        ve = VideoExtractor()
+        ve.name = ''
+        ve.url = urls
+        ve.title=title
     # save download info in streams
     stream = {}
     stream['container'] = ext

From fc1646d74ea14012a03dc17aad395b5c5f1554b3 Mon Sep 17 00:00:00 2001
From: haoflynet <haoflynet@gmail.com>
Date: Sun, 22 Jan 2017 23:35:23 +0800
Subject: [PATCH 0296/1225] fix youku.py bug

---
 src/you_get/extractors/youku.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d673e58c7c..65fcbc2716 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -143,6 +143,9 @@ def prepare(self, **kwargs):
             })
         else:
             proxy_handler = request.ProxyHandler({})
+        if not request._opener:
+            opener = request.build_opener(proxy_handler)
+            request.install_opener(opener)
         for handler in (ssl_context, cookie_handler, proxy_handler):
             request._opener.add_handler(handler)
         request._opener.addheaders = [('Cookie','__ysuid={}'.format(time.time()))]

From 61225b1552df86dbecf1be22c6b5433cd3412f44 Mon Sep 17 00:00:00 2001
From: Chuntao Hong <chuntao.hong@gmail.com>
Date: Tue, 24 Jan 2017 12:36:57 +0800
Subject: [PATCH 0297/1225] fix non-ascii url

---
 src/you_get/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index bea6e62c56..51b81cad5b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -259,6 +259,7 @@ def undeflate(data):
 
 # DEPRECATED in favor of get_content()
 def get_response(url, faker = False):
+    url = parse.quote(url,':/')
     # install cookies
     if cookies:
         opener = request.build_opener(request.HTTPCookieProcessor(cookies))

From 10624ca5b34e542bb9004765889499dc0341d698 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Jan 2017 21:21:09 +0100
Subject: [PATCH 0298/1225] [google] add UA in get_html

---
 src/you_get/extractors/google.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 18483920ca..febac780a3 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -51,7 +51,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         # attempt to extract images first
         # TBD: posts with > 4 images
         # TBD: album links
-        html = get_html(parse.unquote(url))
+        html = get_html(parse.unquote(url), faker=True)
         real_urls = []
         for src in re.findall(r'src="([^"]+)"[^>]*itemprop="image"', html):
             t = src.split('/')
@@ -66,7 +66,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
         try:
             url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
-            html = get_html(url)
+            html = get_html(url, faker=True)
             temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
             temp = sorted(temp, key = lambda x : fmt_level[x[0]])
             urls = [unicodize(i[1]) for i in temp if i[0] == temp[0][0]]
@@ -77,7 +77,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
             post_author = r1(r'/\+([^/]+)/posts', post_url)
             if post_author:
                 post_url = "https://plus.google.com/+%s/posts/%s" % (parse.quote(post_author), r1(r'posts/(.+)', post_url))
-            post_html = get_html(post_url)
+            post_html = get_html(post_url, faker=True)
             title = r1(r'<title[^>]*>([^<\n]+)', post_html)
 
             if title is None:
@@ -98,7 +98,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
     elif service in ['docs', 'drive'] : # Google Docs
 
-        html = get_html(url)
+        html = get_html(url, faker=True)
 
         title = r1(r'"title":"([^"]*)"', html) or r1(r'<meta itemprop="name" content="([^"]*)"', html)
         if len(title.split('.')) > 1:

From f299d30161f2017318211099979845192a891025 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Jan 2017 21:21:49 +0100
Subject: [PATCH 0299/1225] [common] update fake_headers

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index bea6e62c56..9ee38821fd 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -131,7 +131,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0'
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0'
 }
 
 if sys.stdout.isatty():

From 4108e2112deac199fe948fdcf3793148fea3a141 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Thu, 26 Jan 2017 16:31:56 +0800
Subject: [PATCH 0300/1225] fix:[zhanqi.tv]recode all

---
 src/you_get/extractors/zhanqi.py | 99 +++++++++++++-------------------
 1 file changed, 39 insertions(+), 60 deletions(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index 7d6b75b67c..25e7e1329a 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -3,73 +3,52 @@
 __all__ = ['zhanqi_download']
 
 from ..common import *
-import re
-import base64
 import json
-import time
-import hashlib
 
-def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    html = get_content(url)
-    video_type_patt = r'VideoType":"([^"]+)"'
-    video_type = match1(html, video_type_patt)
+def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):#the programmers of zhanqi are noobs
+    host_name = url.split('/')[2]
+    first_folder_path = url.split('/')[3]
+
+    if first_folder_path != 'videos': #url = "https://www.zhanqi.tv/huashan"
+        if first_folder_path == 'topic': #https://www.zhanqi.tv/topic/lyingman
+            first_folder_path = url.split('/')[4]
+        api_url = "https://www.zhanqi.tv/api/static/v2.1/room/domain/" + first_folder_path + ".json"
+        api_json = json.loads(get_html(api_url))
+        data = api_json['data']
+        status = data['status']
+        if status != '4':
+            raise ValueError ("The live stream is not online!")
+
+        nickname = data['nickname']
+        title = nickname + ": " + data['title']
+
+        roomid = data['id']
+        videoId = data['videoId']
+        jump_url = "http://wshdl.load.cdn.zhanqi.tv/zqlive/" + videoId + ".flv?get_url=1"
+        jump_url = jump_url.strip('\r\n')
+
+        real_url = get_html(jump_url)
+        real_url = real_url.strip('\r\n')
 
-    #rtmp_base_patt = r'VideoUrl":"([^"]+)"'
-    rtmp_id_patt = r'videoId":"([^"]+)"'
-    vod_m3u8_id_patt = r'VideoID":"([^"]+)"'
-    title_patt = r'<p class="title-name" title="[^"]+">([^<]+)</p>'
-    title_patt_backup = r'<title>([^<]{1,9999})</title>'
-    title = match1(html, title_patt) or match1(html, title_patt_backup)
-    title = unescape_html(title)
-    rtmp_base = "http://wshdl.load.cdn.zhanqi.tv/zqlive"
-    vod_base = "http://dlvod.cdn.zhanqi.tv"
-    rtmp_real_base = "rtmp://dlrtmp.cdn.zhanqi.tv/zqlive/"
-    room_info = "http://www.zhanqi.tv/api/static/live.roomid/"
-    KEY_MASK = "#{&..?!("
-    ak2_pattern = r'ak2":"\d-([^|]+)'
-    
-    if video_type == "LIVE":
-        rtmp_id = match1(html, rtmp_id_patt).replace('\\/','/')
-        #request_url = rtmp_base+'/'+rtmp_id+'.flv?get_url=1'
-        #real_url = get_html(request_url)
-        html2 = get_content(room_info + rtmp_id.split("_")[0] + ".json")
-        json_data = json.loads(html2)
-        cdns = json_data["data"]["flashvars"]["cdns"]
-        cdns = base64.b64decode(cdns).decode("utf-8")
-        cdn = match1(cdns, ak2_pattern)
-        cdn = base64.b64decode(cdn).decode("utf-8")
-        key = ''
-        i = 0
-        while(i < len(cdn)):
-            key = key + chr(ord(cdn[i]) ^ ord(KEY_MASK[i % 8]))
-            i = i + 1
-        time_hex = hex(int(time.time()))[2:]
-        key = hashlib.md5(bytes(key + "/zqlive/" + rtmp_id + time_hex, "utf-8")).hexdigest()
-        real_url = rtmp_real_base + '/' + rtmp_id + "?k=" + key + "&t=" + time_hex
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
-            download_rtmp_url(real_url, title, 'flv', {}, output_dir, merge = merge)
-            #download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
-    elif video_type == "VOD":
-        vod_m3u8_request = vod_base + match1(html, vod_m3u8_id_patt).replace('\\/','/')
-        vod_m3u8 = get_html(vod_m3u8_request)
-        part_url = re.findall(r'(/[^#]+)\.ts',vod_m3u8)
-        real_url = []
-        for i in part_url:
-            i = vod_base + i + ".ts"
-            real_url.append(i)
-        type_ = ''
-        size = 0
-        for url in real_url:
-            _, type_, temp = url_info(url)
-            size += temp or 0
+            download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
 
-        print_info(site_info, title, type_ or 'ts', size)
+    else: #url = 'https://www.zhanqi.tv/videos/Lyingman/2017/01/182308.html'
+        video_id = url.split('/')[-1].split('.')[0]
+        api_url = "https://www.zhanqi.tv/api/static/v2.1/video/" + video_id + ".json"
+        api_json = json.loads(get_html(api_url))
+        data = api_json['data']
+
+        title = data['title']
+
+        video_url_id = data['flashvars']['VideoID']
+        real_url = "http://dlvod.cdn.zhanqi.tv/" + video_url_id
+
+        print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
-            download_urls(real_url, title, type_ or 'ts', size, output_dir, merge = merge)
-    else:
-        NotImplementedError('Unknown_video_type')
+            download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
 
 site_info = "zhanqi.tv"
 download = zhanqi_download
-download_playlist = playlist_not_supported('zhanqi')
+download_playlist = playlist_not_supported('zhanqi')
\ No newline at end of file

From 15ae8feb5b5e4467e5eed54ff18b32021efaa813 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Sat, 28 Jan 2017 03:08:54 +0800
Subject: [PATCH 0301/1225] little fix

---
 src/you_get/extractors/zhanqi.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index 25e7e1329a..f2c673ca08 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -5,13 +5,13 @@
 from ..common import *
 import json
 
-def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):#the programmers of zhanqi are noobs
+def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     host_name = url.split('/')[2]
-    first_folder_path = url.split('/')[3]
+    first_folder_path = url.split('/')[3].split('?')[0]
 
-    if first_folder_path != 'videos': #url = "https://www.zhanqi.tv/huashan"
+    if first_folder_path != 'videos': #url = "https://www.zhanqi.tv/huashan?param_s=1_0.2.0"
         if first_folder_path == 'topic': #https://www.zhanqi.tv/topic/lyingman
-            first_folder_path = url.split('/')[4]
+            first_folder_path = url.split('/')[4].split('?')[0]
         api_url = "https://www.zhanqi.tv/api/static/v2.1/room/domain/" + first_folder_path + ".json"
         api_json = json.loads(get_html(api_url))
         data = api_json['data']
@@ -29,13 +29,15 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
         real_url = get_html(jump_url)
         real_url = real_url.strip('\r\n')
+        site_info = "www.zhanqi.tv"
 
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
             download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
 
     else: #url = 'https://www.zhanqi.tv/videos/Lyingman/2017/01/182308.html'
-        video_id = url.split('/')[-1].split('.')[0]
+        video_id = url.split('/')[-1].split('?')[0].split('.')[0]
+        assert video_id
         api_url = "https://www.zhanqi.tv/api/static/v2.1/video/" + video_id + ".json"
         api_json = json.loads(get_html(api_url))
         data = api_json['data']
@@ -44,11 +46,11 @@ def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
         video_url_id = data['flashvars']['VideoID']
         real_url = "http://dlvod.cdn.zhanqi.tv/" + video_url_id
+        site_info = "www.zhanqi.tv/videos"
 
         print_info(site_info, title, 'flv', float('inf'))
         if not info_only:
             download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
 
-site_info = "zhanqi.tv"
 download = zhanqi_download
 download_playlist = playlist_not_supported('zhanqi')
\ No newline at end of file

From 753879b49736e314b08c2122ddeef550a06646f8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 Jan 2017 03:20:17 +0100
Subject: [PATCH 0302/1225] [netease] fix #1642

---
 src/you_get/extractors/netease.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index d5f3b1fa9f..17ae70a930 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -22,9 +22,9 @@ def netease_hymn():
     """
 
 def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    rid = match1(url, r'id=(.*)')
+    rid = match1(url, r'\Wid=(.*)')
     if rid is None:
-        rid = match1(url, r'/(\d+)/?$')
+        rid = match1(url, r'/(\d+)/?')
     if "album" in url:
         j = loads(get_content("http://music.163.com/api/album/%s?id=%s&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
 

From 7d72596f06b46299c60ff0c1761cb9167060606a Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Sun, 29 Jan 2017 18:37:10 +0800
Subject: [PATCH 0303/1225] fix quanmin.py

fix #1653
---
 src/you_get/extractors/quanmin.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
index 89d63ea90f..668e84f970 100644
--- a/src/you_get/extractors/quanmin.py
+++ b/src/you_get/extractors/quanmin.py
@@ -7,13 +7,15 @@
 import time
 
 def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    roomid = url[url.rfind("/")+1:]
-    json_request_url = 'http://www.quanmin.tv/json/rooms/{}/info4.json'.format(roomid)
+    roomid = url.split('/')[3].split('?')[0] #add ?parameter.split and change the unstable url(from the back) rfind to split(from the front)
+
+    #json_request_url = 'http://www.quanmin.tv/json/rooms/{}/info4.json'.format(roomid) #"http://www.quanmin.tv/json/rooms/308137/info4.json" switch to "http://m.quanmin.tv/json/rooms/308137/noinfo6.json"
+    json_request_url = 'http://m.quanmin.tv/json/rooms/{}/noinfo6.json'.format(roomid)
     content = get_html(json_request_url)
     data = json.loads(content)
 
     title = data["title"]
-    
+
     if not data["play_status"]:
         raise ValueError("The live stream is not online!")
     real_url = "http://flv.quanmin.tv/live/{}.flv".format(roomid)

From 5139b40b44265128088724a2619f3a3258728517 Mon Sep 17 00:00:00 2001
From: l34p <hmg0228@gmail.com>
Date: Wed, 1 Feb 2017 21:07:59 +0900
Subject: [PATCH 0304/1225] [youtube] fix broken link of html5player

---
 src/you_get/extractors/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index c403cb7400..b0097f137e 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -165,7 +165,7 @@ def prepare(self, **kwargs):
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 try:
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-                    self.html5player = 'https:' + ytplayer_config['assets']['js']
+                    self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                     # Workaround: get_video_info returns bad s. Why?
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 except:
@@ -177,7 +177,7 @@ def prepare(self, **kwargs):
                 ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
 
                 self.title = ytplayer_config['args']['title']
-                self.html5player = 'https:' + ytplayer_config['assets']['js']
+                self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                 stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
 
         elif video_info['status'] == ['fail']:
@@ -193,7 +193,7 @@ def prepare(self, **kwargs):
                     # 150 Restricted from playback on certain sites
                     # Parse video page instead
                     self.title = ytplayer_config['args']['title']
-                    self.html5player = 'https:' + ytplayer_config['assets']['js']
+                    self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 else:
                     log.wtf('[Error] The uploader has not made this video available in your country.')

From 2f4dc0f9a0000ed7ab6ecbfc7d903eed3c71a49d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 1 Feb 2017 17:33:57 +0100
Subject: [PATCH 0305/1225] [google] quick fix for Google+ videos

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index febac780a3..1f2c354c6b 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -65,7 +65,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         title = post_date + "_" + post_id
 
         try:
-            url = "https://plus.google.com/" + r1(r'"(photos/\d+/albums/\d+/\d+)', html)
+            url = "https://plus.google.com/" + r1(r'(photos/\d+/albums/\d+/\d+)\?authkey', html)
             html = get_html(url, faker=True)
             temp = re.findall(r'\[(\d+),\d+,\d+,"([^"]+)"\]', html)
             temp = sorted(temp, key = lambda x : fmt_level[x[0]])

From 8afb998d59be335b4746f1792d317e5f5386a5f1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 1 Feb 2017 23:39:46 +0100
Subject: [PATCH 0306/1225] Remove dead sites (2017-02-01)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* JPopsuki TV http://www.jpopsuki.tv/
* 天天动听 http://www.dongting.com/
* THVideo http://thvideo.tv/
* 阡陌视频 http://qianmo.com/
---
 README.md                          |  4 --
 src/you_get/common.py              |  4 --
 src/you_get/extractors/__init__.py |  3 --
 src/you_get/extractors/dongting.py | 55 --------------------
 src/you_get/extractors/jpopsuki.py | 23 ---------
 src/you_get/extractors/qianmo.py   | 40 --------------
 src/you_get/extractors/thvideo.py  | 83 ------------------------------
 7 files changed, 212 deletions(-)
 delete mode 100644 src/you_get/extractors/dongting.py
 delete mode 100644 src/you_get/extractors/jpopsuki.py
 delete mode 100644 src/you_get/extractors/qianmo.py
 delete mode 100644 src/you_get/extractors/thvideo.py

diff --git a/README.md b/README.md
index 98c403c31b..57f49a6875 100644
--- a/README.md
+++ b/README.md
@@ -347,7 +347,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Metacafe    | <http://www.metacafe.com/>    |✓| | |
 | Magisto     | <http://www.magisto.com/>     |✓| | |
 | Khan Academy | <https://www.khanacademy.org/> |✓| | |
-| JPopsuki TV | <http://www.jpopsuki.tv/>     |✓| | |
 | Internet Archive | <https://archive.org/>   |✓| | |
 | **Instagram** | <https://instagram.com/>    |✓|✓| |
 | InfoQ       | <http://www.infoq.com/presentations/> |✓| | |
@@ -392,11 +391,8 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
 | QQ<br/>腾讯视频 | <http://v.qq.com/>      |✓| | |
 | 企鹅直播 | <http://live.qq.com/>          |✓| | |
-| 阡陌视频 | <http://qianmo.com/>           |✓| | |
-| THVideo  | <http://thvideo.tv/>           |✓| | |
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
 | Sohu<br/>搜狐视频 | <http://tv.sohu.com/> |✓| | |
-| 天天动听 | <http://www.dongting.com/>     | | |✓|
 | **Tudou<br/>土豆** | <http://www.tudou.com/> |✓| | |
 | 虾米     | <http://www.xiami.com/>        | | |✓|
 | 阳光卫视 | <http://www.isuntv.com/>       |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9ee38821fd..a4aea070f5 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -15,7 +15,6 @@
     'cbs'              : 'cbs',
     'dailymotion'      : 'dailymotion',
     'dilidili'         : 'dilidili',
-    'dongting'         : 'dongting',
     'douban'           : 'douban',
     'douyu'            : 'douyutv',
     'ehow'             : 'ehow',
@@ -40,7 +39,6 @@
     'iqiyi'            : 'iqiyi',
     'isuntv'           : 'suntv',
     'joy'              : 'joy',
-    'jpopsuki'         : 'jpopsuki',
     'kankanews'        : 'bilibili',
     'khanacademy'      : 'khan',
     'ku6'              : 'ku6',
@@ -63,7 +61,6 @@
     'pinterest'        : 'pinterest',
     'pixnet'           : 'pixnet',
     'pptv'             : 'pptv',
-    'qianmo'           : 'qianmo',
     'qq'               : 'qq',
     'quanmin'          : 'quanmin',
     'showroom-live'    : 'showroom',
@@ -73,7 +70,6 @@
     'soundcloud'       : 'soundcloud',
     'ted'              : 'ted',
     'theplatform'      : 'theplatform',
-    'thvideo'          : 'thvideo',
     'tucao'            : 'tucao',
     'tudou'            : 'tudou',
     'tumblr'           : 'tumblr',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 61b6a0d180..a027c396d5 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -33,7 +33,6 @@
 from .iqilu import *
 from .iqiyi import *
 from .joy import *
-from .jpopsuki import *
 from .ku6 import *
 from .kugou import *
 from .kuwo import *
@@ -55,7 +54,6 @@
 from .pinterest import *
 from .pixnet import *
 from .pptv import *
-from .qianmo import *
 from .qie import *
 from .qq import *
 from .showroom import *
@@ -64,7 +62,6 @@
 from .soundcloud import *
 from .suntv import *
 from .theplatform import *
-from .thvideo import *
 from .tucao import *
 from .tudou import *
 from .tumblr import *
diff --git a/src/you_get/extractors/dongting.py b/src/you_get/extractors/dongting.py
deleted file mode 100644
index 56c1d3947e..0000000000
--- a/src/you_get/extractors/dongting.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# -*- coding: utf-8 -*-
-
-__all__ = ['dongting_download']
-
-from ..common import *
-
-_unit_prefixes = 'bkmg'
-
-def parse_size(size):
-    m = re.match(r'([\d.]+)(.(?:i?B)?)', size, re.I)
-    if m:
-        return int(float(m.group(1)) * 1024 **
-                   _unit_prefixes.index(m.group(2).lower()))
-    else:
-        return 0
-
-def dongting_download_lyric(lrc_url, file_name, output_dir):
-    j = get_html(lrc_url)
-    info = json.loads(j)
-    lrc = j['data']['lrc']
-    filename = get_filename(file_name)
-    with open(output_dir + "/" + filename + '.lrc', 'w', encoding='utf-8') as x:
-        x.write(lrc)
-
-def dongting_download_song(sid, output_dir = '.', merge = True, info_only = False):
-    j = get_html('http://ting.hotchanson.com/detail.do?neid=%s&size=0' % sid)
-    info = json.loads(j)
-
-    song_title = info['data']['songName']
-    album_name = info['data']['albumName']
-    artist = info['data']['singerName']
-    ext = 'mp3'
-    size = parse_size(info['data']['itemList'][-1]['size'])
-    url = info['data']['itemList'][-1]['downUrl']
-
-    print_info(site_info, song_title, ext, size)
-    if not info_only:
-        file_name = "%s - %s - %s" % (song_title, album_name, artist)
-        download_urls([url], file_name, ext, size, output_dir, merge = merge)
-        lrc_url = ('http://lp.music.ttpod.com/lrc/down?'
-                   'lrcid=&artist=%s&title=%s') % (
-                       parse.quote(artist), parse.quote(song_title))
-        try:
-            dongting_download_lyric(lrc_url, file_name, output_dir)
-        except:
-            pass
-
-def dongting_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, **kwargs):
-    if re.match('http://www.dongting.com/\?song_id=\d+', url):
-        id = r1(r'http://www.dongting.com/\?song_id=(\d+)', url)
-        dongting_download_song(id, output_dir, merge, info_only)
-
-site_info = "Dongting.com"
-download = dongting_download
-download_playlist = playlist_not_supported("dongting")
diff --git a/src/you_get/extractors/jpopsuki.py b/src/you_get/extractors/jpopsuki.py
deleted file mode 100644
index eeac4f6383..0000000000
--- a/src/you_get/extractors/jpopsuki.py
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['jpopsuki_download']
-
-from ..common import *
-
-def jpopsuki_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html(url, faker=True)
-    
-    title = r1(r'<meta name="title" content="([^"]*)"', html)
-    if title.endswith(' - JPopsuki TV'):
-        title = title[:-14]
-    
-    url = "http://jpopsuki.tv%s" % r1(r'<source src="([^"]*)"', html)
-    type, ext, size = url_info(url, faker=True)
-    
-    print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge=merge, faker=True)
-
-site_info = "JPopsuki.tv"
-download = jpopsuki_download
-download_playlist = playlist_not_supported('jpopsuki')
diff --git a/src/you_get/extractors/qianmo.py b/src/you_get/extractors/qianmo.py
deleted file mode 100644
index 7b2b18c182..0000000000
--- a/src/you_get/extractors/qianmo.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['qianmo_download']
-
-from ..common import *
-import urllib.error
-import json
-
-def qianmo_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://qianmo.com/\w+', url):
-        html = get_html(url)
-        match = re.search(r'(.+?)var video =(.+?);', html)
-        
-        if match:
-            video_info_json = json.loads(match.group(2))
-            title = video_info_json['title']
-            ext_video_id = video_info_json['ext_video_id']
-        
-        html = get_content('http://v.qianmo.com/player/{ext_video_id}'.format(ext_video_id = ext_video_id))
-        c = json.loads(html)
-        url_list = []
-        for i in c['seg']:  #Cannot do list comprehensions
-            for a in c['seg'][i]:
-                for b in a['url']:
-                    url_list.append(b[0])
-        
-        type_ = ''
-        size = 0
-        for url in url_list:
-            _, type_, temp = url_info(url)
-            size += temp
-
-        type, ext, size = url_info(url)
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls(url_list, title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-site_info = "qianmo"
-download = qianmo_download
-download_playlist = playlist_not_supported('qianmo')
diff --git a/src/you_get/extractors/thvideo.py b/src/you_get/extractors/thvideo.py
deleted file mode 100644
index 2317b6f5b7..0000000000
--- a/src/you_get/extractors/thvideo.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['thvideo_download']
-
-from ..common import *
-from xml.dom.minidom import parseString
-
-#----------------------------------------------------------------------
-def thvideo_cid_to_url(cid, p):
-    """int,int->list
-    From Biligrab."""
-    interface_url = 'http://thvideo.tv/api/playurl.php?cid={cid}-{p}'.format(cid = cid, p = p)
-    data = get_content(interface_url)
-    rawurl = []
-    dom = parseString(data)
-    
-    for node in dom.getElementsByTagName('durl'):
-        url = node.getElementsByTagName('url')[0]
-        rawurl.append(url.childNodes[0].data)
-    return rawurl
-
-#----------------------------------------------------------------------
-def th_video_get_title(url, p):
-    """"""
-    if re.match(r'http://thvideo.tv/v/\w+', url):
-        html = get_content(url)
-        title = match1(html, r'<meta property="og:title" content="([^"]*)"').strip()
-        
-        video_list = match1(html, r'<li>cid=(.+)</li>').split('**')
-        
-        if int(p) > 0:  #not the 1st P or multi part
-            title = title + ' - ' + [i.split('=')[-1:][0].split('|')[1] for i in video_list][p]
-            
-    return title
-
-#----------------------------------------------------------------------
-def thvideo_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://thvideo.tv/v/\w+', url):
-        if 'p' in kwargs and kwargs['p']:
-            p = kwargs['p']
-        else:
-            p = int(match1(url, r'http://thvideo.tv/v/th\d+#(\d+)'))
-            p -= 1
-            
-            if not p or p < 0:
-                p = 0
-        
-        if 'title' in kwargs and kwargs['title']:
-            title = kwargs['title']
-        else:
-            title = th_video_get_title(url, p)
-        
-        cid = match1(url, r'http://thvideo.tv/v/th(\d+)')
-        
-        type_ = ''
-        size = 0
-        urls = thvideo_cid_to_url(cid, p)
-        
-        for url in urls:
-            _, type_, temp = url_info(url)
-            size += temp
-        
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-#----------------------------------------------------------------------
-def thvideo_download_playlist(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    """"""
-    if re.match(r'http://thvideo.tv/v/\w+', url):
-        html = get_content(url)
-        video_list = match1(html, r'<li>cid=(.+)</li>').split('**')
-        
-        title_base = th_video_get_title(url, 0)
-        for p, v in video_list:
-            part_title = [i.split('=')[-1:][0].split('|')[1] for i in video_list][p]
-            title = title_base + part_title
-            thvideo_download(url, output_dir, merge, 
-                            info_only, p = p, title = title)
-
-site_info = "THVideo"
-download = thvideo_download
-download_playlist = thvideo_download_playlist

From 847e531b0d287d970bcbbdec13b8a2224151b0a8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 1 Feb 2017 23:51:06 +0100
Subject: [PATCH 0307/1225] update .travis.yml (add python 3.6) and LICENSE
 (2017)

---
 .travis.yml | 1 +
 LICENSE.txt | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 9b73708dab..2d780e8181 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,6 +5,7 @@ python:
   - "3.3"
   - "3.4"
   - "3.5"
+  - "3.6"
   - "nightly"
   - "pypy3"
 script: make test
diff --git a/LICENSE.txt b/LICENSE.txt
index 54a06fe54a..7b25d906af 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,7 +1,7 @@
 ==============================================
 This is a copy of the MIT license.
 ==============================================
-Copyright (C) 2012, 2013, 2014, 2015, 2016 Mort Yao <mort.yao@gmail.com>
+Copyright (C) 2012-2017 Mort Yao <mort.yao@gmail.com>
 Copyright (C) 2012 Boyu Guo <iambus@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of

From 62a535a4180a736608e56c440951d8b0e7b23ae8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 1 Feb 2017 23:53:32 +0100
Subject: [PATCH 0308/1225] version 0.4.648

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 2e8e4f414b..933c46ad5f 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.626'
+__version__ = '0.4.648'

From ed99b91d1893186437f52701be03048e50873b9a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 2 Feb 2017 05:43:57 +0100
Subject: [PATCH 0309/1225] [xiami] fix #1650

---
 src/you_get/extractors/xiami.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
index b056c08edc..e321c42e69 100644
--- a/src/you_get/extractors/xiami.py
+++ b/src/you_get/extractors/xiami.py
@@ -13,7 +13,7 @@ def location_dec(str):
     str = str[1:]
     rows = head
     cols = int(len(str)/rows) + 1
-    
+
     out = ""
     full_row = len(str) % head
     for c in range(cols):
@@ -58,7 +58,7 @@ def xiami_download_song(sid, output_dir = '.', merge = True, info_only = False):
     type, ext, size = url_info(url, faker = True)
     if not ext:
         ext = 'mp3'
-    
+
     print_info(site_info, song_title, ext, size)
     if not info_only:
         file_name = "%s - %s - %s" % (song_title, artist, album_name)
@@ -95,7 +95,7 @@ def xiami_download_showcollect(cid, output_dir = '.', merge = True, info_only =
         type, ext, size = url_info(url, faker = True)
         if not ext:
             ext = 'mp3'
-        
+
         print_info(site_info, song_title, type, size)
         if not info_only:
             file_name = "%02d.%s - %s - %s" % (track_nr, song_title, artist, album_name)
@@ -104,7 +104,7 @@ def xiami_download_showcollect(cid, output_dir = '.', merge = True, info_only =
                 xiami_download_lyric(lrc_url, file_name, output_dir)
             except:
                 pass
-        
+
         track_nr += 1
 
 def xiami_download_album(aid, output_dir = '.', merge = True, info_only = False):
@@ -140,22 +140,23 @@ def xiami_download_album(aid, output_dir = '.', merge = True, info_only = False)
             if not pic_exist:
                 xiami_download_pic(pic_url, 'cover', output_dir)
                 pic_exist = True
-        
+
         track_nr += 1
 
 def xiami_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, **kwargs):
     if re.match(r'http://www.xiami.com/album/\d+', url):
         id = r1(r'http://www.xiami.com/album/(\d+)', url)
         xiami_download_album(id, output_dir, merge, info_only)
-    
+
     if re.match(r'http://www.xiami.com/collect/\d+', url):
         id = r1(r'http://www.xiami.com/collect/(\d+)', url)
         xiami_download_showcollect(id, output_dir, merge, info_only)
-    
+
     if re.match('http://www.xiami.com/song/\d+', url):
-        id = r1(r'http://www.xiami.com/song/(\d+)', url)
+        html = get_html(url, faker=True)
+        id = r1(r'rel="canonical" href="http://www.xiami.com/song/([^"]+)"', html)
         xiami_download_song(id, output_dir, merge, info_only)
-    
+
     if re.match('http://www.xiami.com/song/detail/id/\d+', url):
         id = r1(r'http://www.xiami.com/song/detail/id/(\d+)', url)
         xiami_download_song(id, output_dir, merge, info_only)

From 098b6a9dd8b4db5d3516ada1c3dc24fd8d645fba Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 2 Feb 2017 05:50:00 +0100
Subject: [PATCH 0310/1225] [youtube] fix signature extraction

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index b0097f137e..ad1706be63 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -52,7 +52,7 @@ def tr_js(code):
             return code
 
         js = js.replace('\n', ' ')
-        f1 = match1(js, r'\w+\.sig\|\|([$\w]+)\(\w+\.\w+\)')
+        f1 = match1(js, r'"signature",([\w]+)\(\w+\.\w+\)')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)

From c5dbb9766116e6362bd1c3e2a680dedb16979d6f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 2 Feb 2017 06:00:30 +0100
Subject: [PATCH 0311/1225] tests: remove test_freesound

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 020455b044..ba15e4478a 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -8,9 +8,6 @@
 
 class YouGetTests(unittest.TestCase):
 
-    def test_freesound(self):
-        freesound.download("http://www.freesound.org/people/Corsica_S/sounds/184419/", info_only=True)
-
     def test_imgur(self):
         imgur.download("http://imgur.com/WVLk5nD", info_only=True)
         imgur.download("http://imgur.com/gallery/WVLk5nD", info_only=True)

From 858435d5035b72832f5f2a63adbd176916a9a27a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 2 Feb 2017 06:03:23 +0100
Subject: [PATCH 0312/1225] version 0.4.652

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 933c46ad5f..63d908c602 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.648'
+__version__ = '0.4.652'

From b310fdc2d583fb006ebb3a46be10488054ef1561 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Sat, 4 Feb 2017 23:14:33 +0800
Subject: [PATCH 0313/1225] delete comments

---
 src/you_get/extractors/quanmin.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
index 668e84f970..bf1af65976 100644
--- a/src/you_get/extractors/quanmin.py
+++ b/src/you_get/extractors/quanmin.py
@@ -7,9 +7,8 @@
 import time
 
 def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    roomid = url.split('/')[3].split('?')[0] #add ?parameter.split and change the unstable url(from the back) rfind to split(from the front)
+    roomid = url.split('/')[3].split('?')[0]
 
-    #json_request_url = 'http://www.quanmin.tv/json/rooms/{}/info4.json'.format(roomid) #"http://www.quanmin.tv/json/rooms/308137/info4.json" switch to "http://m.quanmin.tv/json/rooms/308137/noinfo6.json"
     json_request_url = 'http://m.quanmin.tv/json/rooms/{}/noinfo6.json'.format(roomid)
     content = get_html(json_request_url)
     data = json.loads(content)

From 69714046b838499c5fce166153ccbf907a69e4a2 Mon Sep 17 00:00:00 2001
From: l34p <hmg0228@gmail.com>
Date: Mon, 6 Feb 2017 18:04:52 +0900
Subject: [PATCH 0314/1225] [youtube] improve livestream detection, fix #1673

---
 src/you_get/extractors/youtube.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ad1706be63..18b46c9d79 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -149,18 +149,6 @@ def prepare(self, **kwargs):
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
                 self.title = parse.unquote_plus(video_info['title'][0])
 
-                # YouTube Live
-                if 'url_encoded_fmt_stream_map' not in video_info:
-                    hlsvp = video_info['hlsvp'][0]
-
-                    if 'info_only' in kwargs and kwargs['info_only']:
-                        return
-                    else:
-                        download_url_ffmpeg(hlsvp, self.title, 'mp4')
-                        exit(0)
-
-                stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
-
                 # Parse video page (for DASH)
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 try:
@@ -169,6 +157,7 @@ def prepare(self, **kwargs):
                     # Workaround: get_video_info returns bad s. Why?
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 except:
+                    stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
                     self.html5player = None
 
             else:
@@ -209,6 +198,16 @@ def prepare(self, **kwargs):
         else:
             log.wtf('[Failed] Invalid status.')
 
+        # YouTube Live
+        if ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1':
+            hlsvp = ytplayer_config['args']['hlsvp']
+
+            if 'info_only' in kwargs and kwargs['info_only']:
+                return
+            else:
+                download_url_ffmpeg(hlsvp, self.title, 'mp4')
+                exit(0)
+
         for stream in stream_list:
             metadata = parse.parse_qs(stream)
             stream_itag = metadata['itag'][0]

From 1997ea45ce2e0afda20f2d9f2f77d9df947d800d Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Thu, 9 Feb 2017 11:26:32 -0500
Subject: [PATCH 0315/1225] [common] log URLs in more functions with network
 requests

This is a follow-up to #999.

This commit adds the

    <function_name>: <url>

debug message, which was previously only emitted by get_content and
post_content, to all high level utility functions with network requests
except url_size, url_save and url_save_chunked (in order not to ruin
progress bars).
---
 src/you_get/common.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a4aea070f5..2edbc4261e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -255,6 +255,8 @@ def undeflate(data):
 
 # DEPRECATED in favor of get_content()
 def get_response(url, faker = False):
+    logging.debug('get_response: %s' % url)
+
     # install cookies
     if cookies:
         opener = request.build_opener(request.HTTPCookieProcessor(cookies))
@@ -275,11 +277,15 @@ def get_response(url, faker = False):
 
 # DEPRECATED in favor of get_content()
 def get_html(url, encoding = None, faker = False):
+    logging.debug('get_html: %s' % url)
+
     content = get_response(url, faker).data
     return str(content, 'utf-8', 'ignore')
 
 # DEPRECATED in favor of get_content()
 def get_decoded_html(url, faker = False):
+    logging.debug('get_decoded_html: %s' % url)
+
     response = get_response(url, faker)
     data = response.data
     charset = r1(r'charset=([\w-]+)', response.headers['content-type'])
@@ -289,6 +295,8 @@ def get_decoded_html(url, faker = False):
         return data
 
 def get_location(url):
+    logging.debug('get_location: %s' % url)
+
     response = request.urlopen(url)
     # urllib will follow redirections and it's too much code to tell urllib
     # not to do that
@@ -394,6 +402,8 @@ def urls_size(urls, faker = False, headers = {}):
     return sum([url_size(url, faker=faker, headers=headers) for url in urls])
 
 def get_head(url, headers = {}, get_method = 'HEAD'):
+    logging.debug('get_head: %s' % url)
+
     if headers:
         req = request.Request(url, headers=headers)
     else:
@@ -403,6 +413,8 @@ def get_head(url, headers = {}, get_method = 'HEAD'):
     return dict(res.headers)
 
 def url_info(url, faker = False, headers = {}):
+    logging.debug('url_info: %s' % url)
+
     if faker:
         response = urlopen_with_retry(request.Request(url, headers=fake_headers))
     elif headers:
@@ -456,6 +468,8 @@ def url_info(url, faker = False, headers = {}):
 def url_locations(urls, faker = False, headers = {}):
     locations = []
     for url in urls:
+        logging.debug('url_locations: %s' % url)
+
         if faker:
             response = urlopen_with_retry(request.Request(url, headers=fake_headers))
         elif headers:

From c1ed0d7e15d327327922235894623f7a551438b3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 12 Feb 2017 00:40:16 +0100
Subject: [PATCH 0316/1225] [youtube] fix signature extraction (regression in
 #1662)

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ad1706be63..6f75a129f1 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -52,7 +52,7 @@ def tr_js(code):
             return code
 
         js = js.replace('\n', ' ')
-        f1 = match1(js, r'"signature",([\w]+)\(\w+\.\w+\)')
+        f1 = match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)

From 4d0dac29681a18520dabe1fc6a6deb81fe20f49d Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Thu, 2 Feb 2017 03:59:44 -0500
Subject: [PATCH 0317/1225] [ffmpeg] call ffmpeg with stdin redirected to the
 null device

Prevent FFmpeg from consuming stdin and interpreting the character
stream as a stream of interactive commands, specifically:

    ?      show this help
    +      increase verbosity
    -      decrease verbosity
    c      Send command to first matching filter supporting it
    C      Send/Queue command to all matching filters
    D      cycle through available debug modes
    h      dump packets/hex press to cycle through the 3 states
    q      quit
    s      Show QP histogram

This prevents misclicking a key or key sequence (e.g., h) produces a
large amount of debugging output which may confuse the unseasoned
user. It is also useful in a batch environment where an unsuspecting
user may not realize you-get could consume stdin through FFmpeg, e.g.

    while read url; do you-get $url; done <urllist

(Related: BashFAQ/089.)

Note that interactive commands are enabled in debug mode (which also
doesn't suppress FFmpeg's default logging).

---

A note on implementation:

An alternative would be FFmpeg's -nostdin option, which explicitly
disables interaction, but adding such an option to existing code would
be even more destructive than the stdin=DEVNULL approach taken here.

---

This commit also happens to fix a dual-call bug in
ffmpeg_concat_mp4_to_mpg.
---
 src/you_get/processor/ffmpeg.py | 38 +++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index da7c076c45..78cd83fa13 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -6,9 +6,18 @@
 from ..util.strings import parameterize
 from ..common import print_more_compatible as print
 
+try:
+    from subprocess import DEVNULL
+except ImportError:
+    # Python 3.2 or below
+    import os
+    import atexit
+    DEVNULL = os.open(os.devnull, os.O_RDWR)
+    atexit.register(lambda fd: os.close(fd), DEVNULL)
+
 def get_usable_ffmpeg(cmd):
     try:
-        p = subprocess.Popen([cmd, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        p = subprocess.Popen([cmd, '-version'], stdin=DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         out, err = p.communicate()
         vers = str(out, 'utf-8').split('\n')[0].split()
         assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
@@ -24,8 +33,10 @@ def get_usable_ffmpeg(cmd):
 FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)
 if logging.getLogger().isEnabledFor(logging.DEBUG):
     LOGLEVEL = ['-loglevel', 'info']
+    STDIN = None
 else:
     LOGLEVEL = ['-loglevel', 'quiet']
+    STDIN = DEVNULL
 
 def has_ffmpeg_installed():
     return FFMPEG is not None
@@ -54,14 +65,14 @@ def ffmpeg_concat_av(files, output, ext):
         params.extend(['-c:a', 'vorbis'])
     params.extend(['-strict', 'experimental'])
     params.append(output)
-    return subprocess.call(params)
+    return subprocess.call(params, stdin=STDIN)
 
 def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
     for file in files:
         if os.path.isfile(file):
             params = [FFMPEG] + LOGLEVEL
             params.extend(['-y', '-i', file, output])
-            subprocess.call(params)
+            subprocess.call(params, stdin=STDIN)
 
     return
 
@@ -71,7 +82,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         concat_list = generate_concat_list(files, output)
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
                                         '-i', concat_list, '-c', 'copy', output]
-        if subprocess.call(params) == 0:
+        if subprocess.call(params, stdin=STDIN) == 0:
             os.remove(output + '.txt')
             return True
         else:
@@ -81,7 +92,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
         if os.path.isfile(file):
             params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
             params.extend([file, file + '.mpg'])
-            subprocess.call(params)
+            subprocess.call(params, stdin=STDIN)
 
     inputs = [open(file + '.mpg', 'rb') for file in files]
     with open(output + '.mpg', 'wb') as o:
@@ -92,9 +103,8 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     params.append(output + '.mpg')
     params += ['-vcodec', 'copy', '-acodec', 'copy']
     params.append(output)
-    subprocess.call(params)
 
-    if subprocess.call(params) == 0:
+    if subprocess.call(params, stdin=STDIN) == 0:
         for file in files:
             os.remove(file + '.mpg')
         os.remove(output + '.mpg')
@@ -112,7 +122,7 @@ def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
     params += ['-f', 'matroska', '-c', 'copy', output]
 
     try:
-        if subprocess.call(params) == 0:
+        if subprocess.call(params, stdin=STDIN) == 0:
             return True
         else:
             return False
@@ -127,7 +137,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
                                         '-i', concat_list, '-c', 'copy',
                                         '-bsf:a', 'aac_adtstoasc', output]
-        subprocess.check_call(params)
+        subprocess.check_call(params, stdin=STDIN)
         os.remove(output + '.txt')
         return True
 
@@ -138,7 +148,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
             params += ['-map', '0', '-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
             params.append(file + '.ts')
 
-            subprocess.call(params)
+            subprocess.call(params, stdin=STDIN)
 
     params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
     params.append('concat:')
@@ -151,7 +161,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     else:
         params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
 
-    if subprocess.call(params) == 0:
+    if subprocess.call(params, stdin=STDIN) == 0:
         for file in files:
             os.remove(file + '.ts')
         return True
@@ -166,7 +176,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
                                         '-i', concat_list, '-c', 'copy',
                                         '-bsf:a', 'aac_adtstoasc', output]
-        subprocess.check_call(params)
+        subprocess.check_call(params, stdin=STDIN)
         os.remove(output + '.txt')
         return True
 
@@ -177,7 +187,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
             params += ['-c', 'copy', '-f', 'mpegts', '-bsf:v', 'h264_mp4toannexb']
             params.append(file + '.ts')
 
-            subprocess.call(params)
+            subprocess.call(params, stdin=STDIN)
 
     params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
     params.append('concat:')
@@ -190,7 +200,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     else:
         params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
 
-    subprocess.check_call(params)
+    subprocess.check_call(params, stdin=STDIN)
     for file in files:
         os.remove(file + '.ts')
     return True

From 8799197befd1f52278a4344fc41ba94cc45c548a Mon Sep 17 00:00:00 2001
From: YK Liu <cos.lyk@gmail.com>
Date: Mon, 20 Feb 2017 15:09:38 +0800
Subject: [PATCH 0318/1225] Print audiolang in json output

---
 src/you_get/json_output.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 3e1bac9f82..0e610a4193 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -11,6 +11,11 @@ def output(video_extractor, pretty_print=True):
     out['title'] = ve.title
     out['site'] = ve.name
     out['streams'] = ve.streams
+    try:
+        if ve.audiolang:
+            out['audiolang'] = ve.audiolang
+    except NameError:
+        pass
     if pretty_print:
         print(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))
     else:

From 9b9d80b32deb6bae475d3d85f376e6d69c6c0835 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 25 Feb 2017 02:31:07 +0800
Subject: [PATCH 0319/1225] do not print size when the container is m3u8

---
 src/you_get/extractor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 332440dd07..af7cc824f2 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -98,7 +98,7 @@ def p_stream(self, stream_id):
         if 'quality' in stream:
             print("      quality:       %s" % stream['quality'])
 
-        if 'size' in stream:
+        if 'size' in stream and stream['container'].lower() != 'm3u8':
             print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
 
         if 'itag' in stream:

From 251a1bff489d2eb34bfa52b54b55dbab6069bd63 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 24 Feb 2017 22:54:59 +0800
Subject: [PATCH 0320/1225] ckplayer.py: fix when got data without a
 ckplayer->info and clean the code

---
 src/you_get/extractors/ckplayer.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/ckplayer.py b/src/you_get/extractors/ckplayer.py
index 09e95557a6..9115989796 100644
--- a/src/you_get/extractors/ckplayer.py
+++ b/src/you_get/extractors/ckplayer.py
@@ -9,7 +9,6 @@
 from xml.etree import cElementTree as ET
 from copy import copy
 from ..common import *
-
 #----------------------------------------------------------------------
 def ckplayer_get_info_by_xml(ckinfo):
     """str->dict
@@ -20,20 +19,22 @@ def ckplayer_get_info_by_xml(ckinfo):
                   'links': [],
                   'size': 0,
                   'flashvars': '',}
-    if '_text' in dictify(e)['ckplayer']['info'][0]['title'][0]:  #title
-        video_dict['title'] = dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip()
+    dictified = dictify(e)['ckplayer']
+    if 'info' in dictified:
+        if '_text' in dictified['info'][0]['title'][0]:  #title
+            video_dict['title'] = dictified['info'][0]['title'][0]['_text'].strip()
 
     #if dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip():  #duration
         #video_dict['title'] = dictify(e)['ckplayer']['info'][0]['title'][0]['_text'].strip()
 
-    if '_text' in dictify(e)['ckplayer']['video'][0]['size'][0]:  #size exists for 1 piece
-        video_dict['size'] = sum([int(i['size'][0]['_text']) for i in dictify(e)['ckplayer']['video']])
+    if '_text' in dictified['video'][0]['size'][0]:  #size exists for 1 piece
+        video_dict['size'] = sum([int(i['size'][0]['_text']) for i in dictified['video']])
 
-    if '_text' in dictify(e)['ckplayer']['video'][0]['file'][0]:  #link exist
-        video_dict['links'] = [i['file'][0]['_text'].strip() for i in dictify(e)['ckplayer']['video']]
+    if '_text' in dictified['video'][0]['file'][0]:  #link exist
+        video_dict['links'] = [i['file'][0]['_text'].strip() for i in dictified['video']]
 
-    if '_text' in dictify(e)['ckplayer']['flashvars'][0]:
-        video_dict['flashvars'] = dictify(e)['ckplayer']['flashvars'][0]['_text'].strip()
+    if '_text' in dictified['flashvars'][0]:
+        video_dict['flashvars'] = dictified['flashvars'][0]['_text'].strip()
 
     return video_dict
 

From 925415fa2b831c6fb5856de0e3739c31c101c1a9 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 25 Feb 2017 00:31:30 +0800
Subject: [PATCH 0321/1225] add support for dilidili.mobi and dilidili.wang

---
 src/you_get/extractors/dilidili.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
index 082f84e130..f7b5922def 100644
--- a/src/you_get/extractors/dilidili.py
+++ b/src/you_get/extractors/dilidili.py
@@ -21,8 +21,9 @@
 #----------------------------------------------------------------------
 def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
     """->list"""
+    another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
     parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
-    html = get_content(parse_url, headers=headers)
+    html = get_content(another_url, headers=headers)
     
     info = re.search(r'(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})', html).groups()
     info = [i.strip('{}').split('->') for i in info]
@@ -35,13 +36,22 @@ def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
 
 #----------------------------------------------------------------------
 def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if re.match(r'http://www.dilidili.com/watch\S+', url):
+    global headers
+    re_str = r'http://www.dilidili.com/watch\S+'
+    if re.match(r'http://www.dilidili.wang', url):
+        re_str = r'http://www.dilidili.wang/watch\S+'
+        headers['Referer'] = 'http://www.dilidili.wang/'
+    elif re.match(r'http://www.dilidili.mobi', url):
+        re_str = r'http://www.dilidili.mobi/watch\S+'
+        headers['Referer'] = 'http://www.dilidili.mobi/'
+
+    if re.match(re_str, url):
         html = get_content(url)
         title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
         
         # player loaded via internal iframe
         frame_url = re.search(r'<iframe src=\"(.+?)\"', html).group(1)
-        #print(frame_url)
+        logging.debug('dilidili_download: %s' % frame_url)
         
         #https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
         html = get_content(frame_url, headers=headers, decoded=False).decode('utf-8')
@@ -53,7 +63,7 @@ def dilidili_download(url, output_dir = '.', merge = False, info_only = False, *
         sign = match1(html, r'var sign="(.+)"')
         tmsign = match1(html, r'tmsign=([A-Za-z0-9]+)')
         ulk =  match1(html, r'var ulk="(.+)"')
-        
+
         # here s the parser...
         stream_types = dilidili_parser_data_to_stream_types(typ, vid, hd2, sign, tmsign, ulk)
         
@@ -62,7 +72,9 @@ def dilidili_download(url, output_dir = '.', merge = False, info_only = False, *
         
         parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = best_id, sign = sign, tmsign = tmsign, ulk = ulk)
         
-        ckplayer_download(parse_url, output_dir, merge, info_only, is_xml = True, title = title, headers = headers)
+        another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
+
+        ckplayer_download(another_url, output_dir, merge, info_only, is_xml = True, title = title, headers = headers)
 
         #type_ = ''
         #size = 0

From d361b55de83354cc631852fdfc9d968e4089b4d0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 26 Feb 2017 23:07:52 +0100
Subject: [PATCH 0322/1225] [nanagogo] skip more

---
 src/you_get/extractors/nanagogo.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/nanagogo.py b/src/you_get/extractors/nanagogo.py
index 9cce9e4c04..1f1ed82003 100644
--- a/src/you_get/extractors/nanagogo.py
+++ b/src/you_get/extractors/nanagogo.py
@@ -19,6 +19,8 @@ def nanagogo_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     items = []
     if info['data']['posts']['post'] is None:
         return
+    if info['data']['posts']['post']['body'] is None:
+        return
     for i in info['data']['posts']['post']['body']:
         if 'image' in i:
             image_url = i['image']

From 37dbb4a3ce4d0ce1b3216ac867ddc704bea5ca32 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Mon, 27 Feb 2017 00:29:45 -0500
Subject: [PATCH 0323/1225] [common] print user agent when dumping URLs (-u,
 --url)

Certain sites (known example: tudou.com) deny access to video URLs unless user
agent matches the one used when retrieving URLs from the API, effectively
rendering the URLs useless without the proper user agent. Therefore, exposing
the user agent in -u, --url output could come in handy at times.

This commit does not affect --json output.
---
 src/you_get/common.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2edbc4261e..b1666f2202 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -751,12 +751,18 @@ def get_output_filename(urls, title, ext, output_dir, merge):
                 merged_ext = 'ts'
     return '%s.%s' % (title, merged_ext)
 
+def print_user_agent(faker=False):
+    urllib_default_user_agent = 'Python-urllib/%d.%d' % sys.version_info[:2]
+    user_agent = fake_headers['User-Agent'] if faker else urllib_default_user_agent
+    print('User Agent: %s' % user_agent)
+
 def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
     assert urls
     if json_output:
         json_output_.download_urls(urls=urls, title=title, ext=ext, total_size=total_size, refer=refer)
         return
     if dry_run:
+        print_user_agent(faker=faker)
         print('Real URLs:\n%s' % '\n'.join(urls))
         return
 
@@ -873,6 +879,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
 def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
     assert urls
     if dry_run:
+        print_user_agent(faker=faker)
         print('Real URLs:\n%s\n' % urls)
         return
 
@@ -952,6 +959,7 @@ def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=No
 def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
     assert url
     if dry_run:
+        print_user_agent(faker=faker)
         print('Real URL:\n%s\n' % [url])
         if params.get("-y",False): #None or unset ->False
             print('Real Playpath:\n%s\n' % [params.get("-y")])
@@ -969,6 +977,7 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re
 def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
     assert url
     if dry_run:
+        print_user_agent(faker=faker)
         print('Real URL:\n%s\n' % [url])
         if params.get("-y",False): #None or unset ->False
             print('Real Playpath:\n%s\n' % [params.get("-y")])

From e4c1d0e23c1ca96e2b33a171a580ea541e227c8d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 1 Mar 2017 02:13:58 +0800
Subject: [PATCH 0324/1225] add .m4a and .DS_Store to .gitignore

---
 .gitignore | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.gitignore b/.gitignore
index d22d3afe3a..0888e5ab95 100644
--- a/.gitignore
+++ b/.gitignore
@@ -81,3 +81,5 @@ _*
 *.xml
 /.env
 /.idea
+*.m4a
+*.DS_Store

From fa9c51dfea4caa6fc1cac44135247373831feca6 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 1 Mar 2017 03:35:47 +0800
Subject: [PATCH 0325/1225] add support for ximalaya.py

---
 src/you_get/common.py              |  3 +-
 src/you_get/extractors/ximalaya.py | 91 ++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+), 1 deletion(-)
 create mode 100644 src/you_get/extractors/ximalaya.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2edbc4261e..c4a71bacbc 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -86,6 +86,7 @@
     'xiami'            : 'xiami',
     'xiaokaxiu'        : 'yixia',
     'xiaojiadianvideo' : 'fc2video',
+    'ximalaya'         : 'ximalaya',
     'yinyuetai'        : 'yinyuetai',
     'miaopai'          : 'yixia',
     'youku'            : 'youku',
@@ -1050,7 +1051,7 @@ def print_info(site_info, title, type, size):
         type_info = "Advanced Systems Format (%s)" % type
     #elif type in ['video/mpeg']:
     #    type_info = "MPEG video (%s)" % type
-    elif type in ['audio/mp4']:
+    elif type in ['audio/mp4', 'audio/m4a']:
         type_info = "MPEG-4 audio (%s)" % type
     elif type in ['audio/mpeg']:
         type_info = "MP3 (%s)" % type
diff --git a/src/you_get/extractors/ximalaya.py b/src/you_get/extractors/ximalaya.py
new file mode 100644
index 0000000000..275b018c01
--- /dev/null
+++ b/src/you_get/extractors/ximalaya.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+__all__ = ['ximalaya_download_playlist', 'ximalaya_download', 'ximalaya_download_by_id']
+
+from ..common import *
+
+import json
+import re
+
+stream_types = [
+        {'itag': '1', 'container': 'm4a', 'bitrate': 'default'},
+        {'itag': '2', 'container': 'm4a', 'bitrate': '32'},
+        {'itag': '3', 'container': 'm4a', 'bitrate': '64'}
+        ]
+def ximalaya_download_by_id(id, title = None, output_dir = '.', info_only = False, stream_id = None):
+    BASE_URL = 'http://www.ximalaya.com/tracks/'
+    json_data = json.loads(get_content(BASE_URL + id + '.json'))
+    if 'res' in json_data:
+        if json_data['res'] == False:
+            raise ValueError('Server reported id %s is invalid' % id)
+    if 'is_paid' in json_data and json_data['is_paid']:
+        raise ValueError('%s is paid item' % id)
+    if (not title) and 'title' in json_data:
+        title = json_data['title']
+#no size data in the json. should it be calculated?
+    size = 0
+    url = json_data['play_path_64']
+    if stream_id:
+        if stream_id == '1':
+            url = json_data['play_path_32']
+        elif stream_id == '0':
+            url = json_data['play_path']
+    logging.debug('ximalaya_download_by_id: %s' % url)
+    ext = 'm4a' 
+    urls = [url]
+    print('Site:        %s' % site_info)
+    print('title:       %s' % title)
+    if info_only:
+        if stream_id:
+            print_stream_info(stream_id)
+        else:
+            for item in range(0, len(stream_types)):
+                print_stream_info(item)
+    if not info_only:
+        print('Type:        MPEG-4 audio m4a')
+        print('Size:        N/A')
+        download_urls(urls, title, ext, size, output_dir = output_dir, merge = False)
+
+def ximalaya_download(url, output_dir = '.', info_only = False, stream_id = None, **kwargs):
+    if re.match(r'http://www\.ximalaya\.com/(\d+)/sound/(\d+)', url):
+        id = match1(url, r'http://www\.ximalaya\.com/\d+/sound/(\d+)')
+    else:
+        raise NotImplementedError(url)
+    ximalaya_download_by_id(id, output_dir = output_dir, info_only = info_only, stream_id = stream_id)
+
+def ximalaya_download_page(playlist_url, output_dir = '.', info_only = False, stream_id = None, **kwargs):
+    if re.match(r'http://www\.ximalaya\.com/(\d+)/album/(\d+)', playlist_url):
+        page_content = get_content(playlist_url)
+        pattern = re.compile(r'<li sound_id="(\d+)"')
+        ids = pattern.findall(page_content)
+        for id in ids:
+            ximalaya_download_by_id(id, output_dir=output_dir, info_only=info_only, stream_id=stream_id)
+    else:
+        raise NotImplementedError(playlist_url)
+def ximalaya_download_playlist(url, output_dir='.', info_only=False, stream_id=None, **kwargs):
+    match_result = re.match(r'http://www\.ximalaya\.com/(\d+)/album/(\d+)', url)
+    if not match_result:
+        raise NotImplementedError(url)
+    pages = []
+    page_content = get_content(url)
+    if page_content.find('<div class="pagingBar_wrapper"') == -1:
+        pages.append(url)
+    else:
+        base_url = 'http://www.ximalaya.com/' + match_result.group(1) + '/album/' + match_result.group(2)
+        html_str = '<a href=(\'|")\/' + match_result.group(1) + '\/album\/' + match_result.group(2) + '\?page='
+        count = len(re.findall(html_str, page_content))
+        for page_num in range(count):
+            pages.append(base_url + '?page=' +str(page_num+1))
+            print(pages[-1])
+    for page in pages:
+        ximalaya_download_page(page, output_dir=output_dir, info_only=info_only, stream_id=stream_id)
+def print_stream_info(stream_id):
+    print('    - itag:        %s' % stream_id)
+    print('      container:   %s' % 'm4a')
+    print('      bitrate:     %s' % stream_types[int(stream_id)]['bitrate'])
+    print('      size:        %s' % 'N/A')
+    print('    # download-with: you-get --itag=%s [URL]' % stream_id)
+
+site_info = 'ximalaya.com'
+download = ximalaya_download
+download_playlist = ximalaya_download_playlist 

From 36bdff7c16256a36aec5964bbf14a6358b533bf1 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 1 Mar 2017 03:56:43 +0800
Subject: [PATCH 0326/1225] [ximalaya]ignore paid item in an album

---
 src/you_get/extractors/ximalaya.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ximalaya.py b/src/you_get/extractors/ximalaya.py
index 275b018c01..e91a1c4803 100644
--- a/src/you_get/extractors/ximalaya.py
+++ b/src/you_get/extractors/ximalaya.py
@@ -12,6 +12,7 @@
         {'itag': '2', 'container': 'm4a', 'bitrate': '32'},
         {'itag': '3', 'container': 'm4a', 'bitrate': '64'}
         ]
+
 def ximalaya_download_by_id(id, title = None, output_dir = '.', info_only = False, stream_id = None):
     BASE_URL = 'http://www.ximalaya.com/tracks/'
     json_data = json.loads(get_content(BASE_URL + id + '.json'))
@@ -59,9 +60,13 @@ def ximalaya_download_page(playlist_url, output_dir = '.', info_only = False, st
         pattern = re.compile(r'<li sound_id="(\d+)"')
         ids = pattern.findall(page_content)
         for id in ids:
-            ximalaya_download_by_id(id, output_dir=output_dir, info_only=info_only, stream_id=stream_id)
+            try:
+                ximalaya_download_by_id(id, output_dir=output_dir, info_only=info_only, stream_id=stream_id)
+            except(ValueError):
+                print("something wrong with %s, perhaps paid item?" % id)
     else:
         raise NotImplementedError(playlist_url)
+    
 def ximalaya_download_playlist(url, output_dir='.', info_only=False, stream_id=None, **kwargs):
     match_result = re.match(r'http://www\.ximalaya\.com/(\d+)/album/(\d+)', url)
     if not match_result:

From 63eca70f86bfd7d56c28c16ffae79fc91da558eb Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 1 Mar 2017 17:08:22 +0800
Subject: [PATCH 0327/1225] [ximalaya]support downloading (is_paid and is_free)
 tracks

---
 src/you_get/extractors/ximalaya.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ximalaya.py b/src/you_get/extractors/ximalaya.py
index e91a1c4803..58a158ba57 100644
--- a/src/you_get/extractors/ximalaya.py
+++ b/src/you_get/extractors/ximalaya.py
@@ -20,7 +20,8 @@ def ximalaya_download_by_id(id, title = None, output_dir = '.', info_only = Fals
         if json_data['res'] == False:
             raise ValueError('Server reported id %s is invalid' % id)
     if 'is_paid' in json_data and json_data['is_paid']:
-        raise ValueError('%s is paid item' % id)
+        if 'is_free' in json_data and not json_data['is_free']:
+            raise ValueError('%s is paid item' % id)
     if (not title) and 'title' in json_data:
         title = json_data['title']
 #no size data in the json. should it be calculated?

From bea95ed3d30cceca710bf523599cf19b6faf53d3 Mon Sep 17 00:00:00 2001
From: steven7851 <steven7851@msn.com>
Date: Sun, 5 Mar 2017 20:17:46 +0800
Subject: [PATCH 0328/1225] use new api

from https://gist.github.com/spacemeowx2/629b1d131bd7e240a7d28742048e80fc
https://github.com/soimort/you-get/issues/1720
---
 src/you_get/extractors/douyutv.py | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 84aeb00f50..2f144ffa22 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -6,8 +6,6 @@
 import json
 import hashlib
 import time
-import uuid
-import urllib.parse, urllib.request
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html = get_content(url)
@@ -28,24 +26,19 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     if show_status is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
-    tt = int(time.time() / 60)
-    did = uuid.uuid4().hex.upper()
-    sign_content = '{room_id}{did}A12Svb&%1UUmf@hC{tt}'.format(room_id = room_id, did = did, tt = tt)
-    sign = hashlib.md5(sign_content.encode('utf-8')).hexdigest()
+    tt = int(time.time())
+    sign_content = 'lapi/live/thirdPart/getPlay/%s?aid=pcclient&rate=0&time=%s9TUk5fjjUjg9qIMH3sdnh' % (room_id, tt)
+    sign = hashlib.md5(sign_content.encode('ascii')).hexdigest()
 
-    json_request_url = "http://www.douyu.com/lapi/live/getPlay/%s" % room_id
-    payload = {'cdn': 'ws', 'rate': '0', 'tt': tt, 'did': did, 'sign': sign}
-    postdata = urllib.parse.urlencode(payload)
-    req = urllib.request.Request(json_request_url, postdata.encode('utf-8'))
-    with urllib.request.urlopen(req) as response:
-        content = response.read()
-
-    data = json.loads(content.decode('utf-8'))['data']
+    json_request_url = "http://coapi.douyucdn.cn/lapi/live/thirdPart/getPlay/%s?rate=0" % room_id
+    headers = {'auth': sign, 'time': str(tt), 'aid': 'pcclient'}
+    content = get_content(json_request_url, headers = headers)
+    data = json.loads(content)['data']
     server_status = data.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
 
-    real_url = data.get('rtmp_url')+'/'+data.get('rtmp_live')
+    real_url = data.get('live_url')
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From 96398d8f6c2ce95c19324d6ea38d89e6c15c71fb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 5 Mar 2017 23:58:00 +0100
Subject: [PATCH 0329/1225] [common] revert #1644 (per #1728)

---
 src/you_get/common.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index be3ee3fd08..c4a71bacbc 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -257,7 +257,6 @@ def undeflate(data):
 # DEPRECATED in favor of get_content()
 def get_response(url, faker = False):
     logging.debug('get_response: %s' % url)
-    url = parse.quote(url, ':/')
 
     # install cookies
     if cookies:
@@ -280,7 +279,6 @@ def get_response(url, faker = False):
 # DEPRECATED in favor of get_content()
 def get_html(url, encoding = None, faker = False):
     logging.debug('get_html: %s' % url)
-    url = parse.quote(url, ':/')
 
     content = get_response(url, faker).data
     return str(content, 'utf-8', 'ignore')
@@ -288,7 +286,6 @@ def get_html(url, encoding = None, faker = False):
 # DEPRECATED in favor of get_content()
 def get_decoded_html(url, faker = False):
     logging.debug('get_decoded_html: %s' % url)
-    url = parse.quote(url, ':/')
 
     response = get_response(url, faker)
     data = response.data
@@ -300,7 +297,6 @@ def get_decoded_html(url, faker = False):
 
 def get_location(url):
     logging.debug('get_location: %s' % url)
-    url = parse.quote(url, ':/')
 
     response = request.urlopen(url)
     # urllib will follow redirections and it's too much code to tell urllib
@@ -327,7 +323,6 @@ def get_content(url, headers={}, decoded=True):
     """
 
     logging.debug('get_content: %s' % url)
-    url = parse.quote(url, ':/')
 
     req = request.Request(url, headers=headers)
     if cookies:
@@ -367,7 +362,6 @@ def post_content(url, headers={}, post_data={}, decoded=True):
     """
 
     logging.debug('post_content: %s \n post_data: %s' % (url, post_data))
-    url = parse.quote(url, ':/')
 
     req = request.Request(url, headers=headers)
     if cookies:

From 86493087d4b467865d6f18ec2293c0c0cfa243a0 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 3 Mar 2017 08:03:23 +0800
Subject: [PATCH 0330/1225] [common.py] switch to gnu_getopt

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c4a71bacbc..9c7a9e6f55 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1193,7 +1193,7 @@ def version():
         opts = ['playlist'] + opts
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], short_opts, opts)
+        opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, opts)
     except getopt.GetoptError as err:
         log.e(err)
         log.e("try 'you-get --help' for more options")

From 0a8f708c31888cc1645ef6072edbf99ae17a9187 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 6 Mar 2017 11:34:50 +0800
Subject: [PATCH 0331/1225] [common]add -I --input-file flag

---
 src/you_get/common.py | 27 +++++++++++++++++++++------
 1 file changed, 21 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9c7a9e6f55..56fb305fb7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1184,13 +1184,15 @@ def version():
     -s | --socks-proxy <HOST:PORT>      Use an SOCKS5 proxy for downloading.
     -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
+    -I | --input-file                   Read non-playlist urls from file.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=']
-    if download_playlist:
-        short_opts = 'l' + short_opts
-        opts = ['playlist'] + opts
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=', 'input-file=']
+#dead code? download_playlist is a function and always True
+#if download_playlist:
+    short_opts = 'l' + short_opts
+    opts = ['playlist'] + opts
 
     try:
         opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, opts)
@@ -1219,6 +1221,8 @@ def version():
     extractor_proxy = None
     traceback = False
     timeout = 600
+    urls_from_file = []
+
     for o, a in opts:
         if o in ('-V', '--version'):
             version()
@@ -1296,12 +1300,23 @@ def version():
             lang = a
         elif o in ('-t', '--timeout'):
             timeout = int(a)
+        elif o in ('-I', '--input-file'):
+            logging.debug('you are trying to load urls from {}'.format(a))
+            if playlist:
+                log.e("reading playlist from a file is unsupported and won't make your life easier")
+                sys.exit(2)
+            with open(a, 'r') as input_file:
+                for line in input_file:
+                    url = line.strip()
+                    urls_from_file.append(url)
         else:
             log.e("try 'you-get --help' for more options")
             sys.exit(2)
-    if not args:
+    if not args and not urls_from_file:
         print(help)
         sys.exit()
+    args.extend(urls_from_file)
+    print(args)
 
     if (socks_proxy):
         try:

From db6fa04754a354f46931ef9ac03878eefcac9fcd Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 6 Mar 2017 11:41:17 +0800
Subject: [PATCH 0332/1225] remove test code

---
 src/you_get/common.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 56fb305fb7..9ad96cae82 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1316,7 +1316,6 @@ def version():
         print(help)
         sys.exit()
     args.extend(urls_from_file)
-    print(args)
 
     if (socks_proxy):
         try:

From cedab9419dbf9a69eb1989fcda458e73a1195c93 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Wed, 8 Mar 2017 20:47:03 -0500
Subject: [PATCH 0333/1225] youku: warn about segments skipped due to paywall

This is especially helpful in cases where the entire video is blocked by
paywall, resulting in an unhelpful error message "you-get: [Failed]
Cannot extract video source."
---
 src/you_get/extractors/youku.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 65fcbc2716..ff23e70652 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -295,9 +295,14 @@ def extract(self, **kwargs):
                 for piece in pieces:
                     segs = piece['segs']
                     streamfileid = piece['fileid']
-                    for no in range(0, len(segs)):
+                    seg_count = len(segs)
+                    for no in range(0, seg_count):
                         k = segs[no]['key']
-                        if k == -1: break # we hit the paywall; stop here
+                        if k == -1:
+                            # we hit the paywall; stop here
+                            log.w('Skipping %d out of %d segments due to paywall' %
+                                  (seg_count - no, seg_count))
+                            break
                         fileid, ep = self.__class__.generate_ep(self, no, streamfileid,
                                                                 sid, token)
                         q = parse.urlencode(dict(

From 4255cc8b86fad2c062992716d2cdaf17d4923f15 Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Wed, 8 Mar 2017 20:59:21 -0500
Subject: [PATCH 0334/1225] log: mark xterm* terminals as ANSI escape
 sequences-compatible

xterm-color, xterm-16color, xterm-88color and xterm-256color are now
covered.
---
 src/you_get/util/log.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
 mode change 100644 => 100755 src/you_get/util/log.py

diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
old mode 100644
new mode 100755
index 5c8504f50c..b858789ccd
--- a/src/you_get/util/log.py
+++ b/src/you_get/util/log.py
@@ -5,13 +5,13 @@
 
 import os, sys
 
-IS_ANSI_TERMINAL = os.getenv('TERM') in (
+TERM = os.getenv('TERM', '')
+IS_ANSI_TERMINAL = TERM in (
     'eterm-color',
     'linux',
     'screen',
     'vt100',
-    'xterm',
-)
+) or TERM.startswith('xterm')
 
 # ANSI escape code
 # See <http://en.wikipedia.org/wiki/ANSI_escape_code>

From ee93e12675565550deef8e5d517cbeb75fe95d55 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 9 Mar 2017 20:47:35 +0800
Subject: [PATCH 0335/1225] [ku6] add baidu.ku6 matching pattern

---
 src/you_get/extractors/ku6.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index 7f28c75bcc..808619f439 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -36,7 +36,6 @@ def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwarg
                 r'http://v.ku6.com/show/(.*)\.\.\.html',
                 r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
         id = r1_of(patterns, url)
-
     ku6_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def baidu_ku6(url):
@@ -48,6 +47,10 @@ def baidu_ku6(url):
     if isrc is not None:
         h2 = get_html(isrc)
         id = match1(h2, r'http://v.ku6.com/show/(.*)\.\.\.html')
+#fix #1746
+#some ku6 urls really ends with three dots? A bug?
+        if id is None:
+            id = match1(h2, r'http://v.ku6.com/show/(.*)\.html')
 
     return id
 

From 3d2d748e72c1480bcae28c045a8f6caa4d044a33 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 11 Mar 2017 15:35:14 +0800
Subject: [PATCH 0336/1225] [qq] add support for kg.qq.com

---
 src/you_get/extractors/qq.py | 43 ++++++++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index f2c3d9ece1..b462bc85c8 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -69,9 +69,52 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
             if not info_only:
                 download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
+def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=False):
+    BASE_URL = 'http://cgi.kg.qq.com/fcgi-bin/kg_ugc_getdetail'
+    params_str = '?dataType=jsonp&jsonp=callback&jsonpCallback=jsopgetsonginfo&v=4&outCharset=utf-8&shareid=' + shareid
+    url = BASE_URL + params_str
+    content = get_content(url)
+    json_str = content[len('jsonpcallback('):-1]
+    json_data = json.loads(json_str)
+
+    playurl = json_data['data']['playurl']
+    videourl = json_data['data']['playurl_video']
+    real_url = playurl if playurl else videourl
+    real_url = real_url.replace('\/', '/')
+
+    ksong_mid = json_data['data']['ksong_mid']
+    lyric_url = 'http://cgi.kg.qq.com/fcgi-bin/fcg_lyric?jsonpCallback=jsopgetlrcdata&outCharset=utf-8&ksongmid=' + ksong_mid 
+    lyric_data = get_content(lyric_url)
+    lyric_string = lyric_data[len('jsopgetlrcdata('):-1]
+    lyric_json = json.loads(lyric_string)
+    lyric = lyric_json['data']['lyric']
+
+    title = match1(lyric, r'\[ti:([^\]]*)\]')
+
+    type, ext, size = url_info(real_url)
+    if not title:
+        title = shareid
+
+    print_info('腾讯全民K歌', title, type, size)
+    if not info_only:
+        download_urls([real_url], title, ext, size, output_dir, merge=False)
+        if caption:
+            caption_filename = title + '.lrc'
+            caption_path = output_dir + '/' + caption_filename
+            with open(caption_path, 'w') as f:
+                lrc_list = lyric.split('\r\n')
+                for line in lrc_list:
+                    f.write(line)
+                    f.write('\n')
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
+    if 'kg.qq.com' in url or 'kg2.qq.com' in url:
+        shareid = url.split('?s=')[-1]
+        caption = kwargs['caption']
+        kg_qq_download_by_shareid(shareid, output_dir=output_dir, info_only=info_only, caption=caption)
+        return
+
     if 'live.qq.com' in url:
         qieDownload(url, output_dir=output_dir, merge=merge, info_only=info_only)
         return

From 89f2331a0cb4255243a7f16f3af7284092bc8acd Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 11 Mar 2017 21:56:30 +0800
Subject: [PATCH 0337/1225] [xiami] fix #1650 again

example url: http://www.xiami.com/song/xL4jJGd3829
---
 src/you_get/extractors/xiami.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
index e321c42e69..3b915b4fa2 100644
--- a/src/you_get/extractors/xiami.py
+++ b/src/you_get/extractors/xiami.py
@@ -152,7 +152,10 @@ def xiami_download(url, output_dir = '.', stream_type = None, merge = True, info
         id = r1(r'http://www.xiami.com/collect/(\d+)', url)
         xiami_download_showcollect(id, output_dir, merge, info_only)
 
-    if re.match('http://www.xiami.com/song/\d+', url):
+    if re.match(r'http://www.xiami.com/song/\d+\b', url):
+        id = r1(r'http://www.xiami.com/song/(\d+)', url)
+        xiami_download_song(id, output_dir, merge, info_only)
+    elif re.match(r'http://www.xiami.com/song/\w+', url):
         html = get_html(url, faker=True)
         id = r1(r'rel="canonical" href="http://www.xiami.com/song/([^"]+)"', html)
         xiami_download_song(id, output_dir, merge, info_only)

From 5d1293d4a4ca8f72082a93682c9a593bc496a6ed Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sat, 11 Mar 2017 21:59:45 +0800
Subject: [PATCH 0338/1225] [xiami] title is wrong, use name instead

example url: http://www.xiami.com/song/xL4jJGd3829
---
 src/you_get/extractors/xiami.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
index 3b915b4fa2..ff9656022d 100644
--- a/src/you_get/extractors/xiami.py
+++ b/src/you_get/extractors/xiami.py
@@ -49,7 +49,7 @@ def xiami_download_song(sid, output_dir = '.', merge = True, info_only = False):
     i = doc.getElementsByTagName("track")[0]
     artist = i.getElementsByTagName("artist")[0].firstChild.nodeValue
     album_name = i.getElementsByTagName("album_name")[0].firstChild.nodeValue
-    song_title = i.getElementsByTagName("title")[0].firstChild.nodeValue
+    song_title = i.getElementsByTagName("name")[0].firstChild.nodeValue
     url = location_dec(i.getElementsByTagName("location")[0].firstChild.nodeValue)
     try:
         lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue

From 3adc839870e98cc0dcb54831691ef3c23f1a777d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 12 Mar 2017 15:56:06 +0800
Subject: [PATCH 0339/1225] [mgtv] add url patterns

---
 src/you_get/extractors/mgtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 1656ac3c60..b0df6b28bc 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -27,7 +27,10 @@ class MGTV(VideoExtractor):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        return match1(url, 'http://www.mgtv.com/b/\d+/(\d+).html')
+        vid = match1(url, 'http://www.mgtv.com/b/\d+/(\d+).html')
+        if not vid:
+            vid = match1(url, 'http://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
+        return vid
     
     #----------------------------------------------------------------------
     @staticmethod
@@ -150,4 +153,4 @@ def download(self, **kwargs):
 
 site = MGTV()
 download = site.download_by_url
-download_playlist = site.download_playlist_by_url
\ No newline at end of file
+download_playlist = site.download_playlist_by_url

From 027d13c97eb31f2b9d31eceb376b7428671b6b23 Mon Sep 17 00:00:00 2001
From: up1m <up1m@163.com>
Date: Fri, 17 Mar 2017 02:03:55 +0800
Subject: [PATCH 0340/1225] make video.weibo.com downloadable

---
 src/you_get/extractors/miaopai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 48a712392d..72dfdaf705 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -19,7 +19,7 @@ def miaopai_download_by_url(url, output_dir = '.', merge = False, info_only = Fa
 
         #grab download URL
         a = get_content(webpage_url, headers= fake_headers_mobile , decoded=True)
-        url = match1(a, r'<video src="(.*?)\"\W')
+        url = match1(a, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
 
         #grab title
         b = get_content(webpage_url)  #normal
@@ -28,7 +28,7 @@ def miaopai_download_by_url(url, output_dir = '.', merge = False, info_only = Fa
         type_, ext, size = url_info(url)
         print_info(site_info, title, type_, size)
         if not info_only:
-            download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
+            download_urls([url], title.replace('\n',''), ext, total_size=None, output_dir=output_dir, merge=merge)
 
 #----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):

From 2b7a5419cbfa1596e19e0f17a6acc29a7601f1a1 Mon Sep 17 00:00:00 2001
From: YK Liu <cos.lyk@gmail.com>
Date: Sat, 18 Mar 2017 13:24:00 +0800
Subject: [PATCH 0341/1225] Change "NameError" to "AttributeError"

---
 src/you_get/json_output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 0e610a4193..0f0f48ec51 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -14,7 +14,7 @@ def output(video_extractor, pretty_print=True):
     try:
         if ve.audiolang:
             out['audiolang'] = ve.audiolang
-    except NameError:
+    except AttributeError:
         pass
     if pretty_print:
         print(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))

From 85b0abf51704b795eefa6fdf1a14bf50727a0aa8 Mon Sep 17 00:00:00 2001
From: YK Liu <cos.lyk@gmail.com>
Date: Wed, 22 Mar 2017 19:16:48 +0800
Subject: [PATCH 0342/1225] Fix qq video; #1778

---
 src/you_get/extractors/qq.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index f2c3d9ece1..c92b730193 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -14,6 +14,8 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     parts_ti = video_json['vl']['vi'][0]['ti']
     parts_prefix = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
     parts_formats = video_json['fl']['fi']
+    if parts_prefix.endswith('/'):
+        parts_prefix = parts_prefix[:-1]
     # find best quality
     # only looking for fhd(1080p) and shd(720p) here.
     # 480p usually come with a single file, will be downloaded as fallback.
@@ -38,7 +40,7 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
                 # For fhd(1080p), every part is about 100M and 6 minutes
                 # try 100 parts here limited download longest single video of 10 hours.
                 for part in range(1,100):
-                    filename = vid + '.p' + str(part_format_id % 1000) + '.' + str(part) + '.mp4'
+                    filename = vid + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
                     key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format=%s&vid=%s&filename=%s" % (part_format_id, parts_vid, filename)
                     #print(filename)
                     #print(key_api)
@@ -59,7 +61,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
             fvkey = video_json['vl']['vi'][0]['fvkey']
             mp4 = video_json['vl']['vi'][0]['cl'].get('ci', None)
             if mp4:
-                mp4 = mp4[0]['keyid'].replace('.10', '.p') + '.mp4'
+                old_id = mp4[0]['keyid'].split('.')[1]
+                new_id = 'p' + str(int(old_id) % 10000)
+                mp4 = mp4[0]['keyid'].replace(old_id, new_id) + '.mp4'
             else:
                 mp4 = video_json['vl']['vi'][0]['fn']
             url = '%s/%s?vkey=%s' % ( parts_prefix, mp4, fvkey )

From 24e59bae1972f45ba832c6c6923f36ab2be4e32e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 23 Mar 2017 00:16:36 +0800
Subject: [PATCH 0343/1225] [vine]fix vine

---
 src/you_get/extractors/vine.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/vine.py b/src/you_get/extractors/vine.py
index 5c54c5fdb9..491ced73ff 100644
--- a/src/you_get/extractors/vine.py
+++ b/src/you_get/extractors/vine.py
@@ -3,6 +3,7 @@
 __all__ = ['vine_download']
 
 from ..common import *
+import json
 
 def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
@@ -11,7 +12,17 @@ def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = r1(r'<title>([^<]*)</title>', html)
     stream = r1(r'<meta property="twitter:player:stream" content="([^"]*)">', html)
     if not stream: # https://vine.co/v/.../card
-        stream = r1(r'"videoUrl":"([^"]+)"', html).replace('\\/', '/')
+        stream = r1(r'"videoUrl":"([^"]+)"', html)
+        if stream:
+            stream = stream.replace('\\/', '/')
+        else:
+            if url[-1] == '/':
+                url = url[:-1]
+            video_id = url.split('/')[-1]
+            posts_url = 'https://archive.vine.co/posts/' + video_id + '.json'
+            json_data = json.loads(get_content(posts_url))
+            stream = json_data['videoDashUrl']
+            title = json_data['description']
 
     mime, ext, size = url_info(stream)
 

From 39bcdfc4a2e8ab05dcd46d9086e96acfe64c987b Mon Sep 17 00:00:00 2001
From: Ein Verne <einverne@gmail.com>
Date: Sat, 1 Apr 2017 14:39:41 +0800
Subject: [PATCH 0344/1225] Fix cntv

---
 src/you_get/extractors/cntv.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index e25fa96116..87f1984f9d 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -32,6 +32,8 @@ def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
 def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
+    elif re.match(r'http://tv\.cctv\.com/\d+/\d+/\d+/\w+.shtml', url):
+        id = r1(r'var guid = "(\w+)"', get_html(url))
     elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
          re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \

From 2d6c828ad4fa9d449ca38fad2cdd4c3c12da27e9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 1 Apr 2017 22:14:13 +0200
Subject: [PATCH 0345/1225] util/log.py: change mode back to 644

---
 src/you_get/util/log.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/util/log.py

diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
old mode 100755
new mode 100644

From f0682403c022b89b98bac408878d5518ac939ff7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 24 Mar 2017 20:42:48 +0100
Subject: [PATCH 0346/1225] [common] allow specifying timeout for url_save()

---
 src/you_get/common.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 78594f176d..45e277a81b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -482,7 +482,7 @@ def url_locations(urls, faker = False, headers = {}):
         locations.append(response.url)
     return locations
 
-def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}):
+def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}, timeout = None, **kwargs):
     file_size = url_size(url, faker = faker, headers = headers)
 
     if os.path.exists(filepath):
@@ -527,7 +527,10 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
         if refer:
             headers['Referer'] = refer
 
-        response = urlopen_with_retry(request.Request(url, headers=headers))
+        if timeout:
+            response = urlopen_with_retry(request.Request(url, headers=headers), timeout=timeout)
+        else:
+            response = urlopen_with_retry(request.Request(url, headers=headers))
         try:
             range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
             end_length = int(response.headers['content-range'][6:].split('/')[1])
@@ -791,7 +794,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
         url = urls[0]
         print('Downloading %s ...' % tr(output_filename))
         bar.update()
-        url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers)
+        url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
         bar.done()
     else:
         parts = []
@@ -803,7 +806,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
             parts.append(filepath)
             #print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
             bar.update_piece(i + 1)
-            url_save(url, filepath, bar, refer = refer, is_part = True, faker = faker, headers = headers)
+            url_save(url, filepath, bar, refer = refer, is_part = True, faker = faker, headers = headers, **kwargs)
         bar.done()
 
         if not merge:

From 5525694fe3fa4274515184d56675a9b3fdf81092 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 24 Mar 2017 20:44:17 +0100
Subject: [PATCH 0347/1225] [bilibili] workaround for speed limiting issue, fix
 #1784

---
 src/you_get/extractors/bilibili.py | 38 +++++++++++++++++-------------
 1 file changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 47dfe04540..a776f9a612 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -86,22 +86,28 @@ def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only
 
 
 def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
-    url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
-    urls = [i
-            if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-            else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-            for i in parse_cid_playurl(get_content(url))]
-
-    type_ = ''
-    size = 0
-    for url in urls:
-        _, type_, temp = url_info(url)
-        size += temp or 0
-
-    print_info(site_info, title, type_, size)
-    if not info_only:
-        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
+    while True:
+        try:
+            sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
+            url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
+            urls = [i
+                    if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
+                    else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
+                    for i in parse_cid_playurl(get_content(url))]
+
+            type_ = ''
+            size = 0
+            for url in urls:
+                _, type_, temp = url_info(url)
+                size += temp or 0
+
+            print_info(site_info, title, type_, size)
+            if not info_only:
+                download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, timeout=1)
+        except socket.timeout:
+            continue
+        else:
+            break
 
 
 def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):

From aaa96f4389281451c84ac2532dfd3a2fa2854f64 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 5 Apr 2017 21:12:39 +0800
Subject: [PATCH 0348/1225] add ifeng new matching patterns

---
 src/you_get/extractors/ifeng.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index 6ca45b30eb..b1b8524a2e 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -25,8 +25,12 @@ def ifeng_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     if id:
         return ifeng_download_by_id(id, None, output_dir = output_dir, merge = merge, info_only = info_only)
 
-    html = get_html(url)
+    html = get_content(url)
+    uuid_pattern = r'"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"'
     id = r1(r'var vid="([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})"', html)
+    if id is None:
+        video_pattern = r'"vid"\s*:\s*' + uuid_pattern
+        id = match1(html, video_pattern)
     assert id, "can't find video info"
     return ifeng_download_by_id(id, None, output_dir = output_dir, merge = merge, info_only = info_only)
 

From 0e87a5c06c216ba151e7a71a6bc03101f790688d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 7 Apr 2017 11:09:28 +0800
Subject: [PATCH 0349/1225] common.py append ext name when -O flag existed

---
 src/you_get/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 45e277a81b..a811cd3570 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -737,7 +737,10 @@ def done(self):
 def get_output_filename(urls, title, ext, output_dir, merge):
     # lame hack for the --output-filename option
     global output_filename
-    if output_filename: return output_filename
+    if output_filename:
+        if ext:
+            return output_filename + '.' + ext
+        return output_filename
 
     merged_ext = ext
     if (len(urls) > 1) and merge:

From b90884cbdf35f2bfe1f0a43f36830f49655a1949 Mon Sep 17 00:00:00 2001
From: Adam Cavendish <GetbetterABC@yeah.net>
Date: Sat, 15 Apr 2017 23:47:40 +0800
Subject: [PATCH 0350/1225] Fix youku's problem, issue #1856

---
 src/you_get/extractors/youku.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index ff23e70652..f36d633947 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -228,14 +228,14 @@ def prepare(self, **kwargs):
                         'video_profile': stream_types[stream_id]['video_profile'],
                         'size': stream['size'],
                         'pieces': [{
-                            'fileid': stream['stream_fileid'],
+                            'fileid': stream['segs'][0]['fileid'],
                             'segs': stream['segs']
                         }]
                     }
                 else:
                     self.streams[stream_id]['size'] += stream['size']
                     self.streams[stream_id]['pieces'].append({
-                        'fileid': stream['stream_fileid'],
+                        'fileid': stream['segs'][0]['fileid'],
                         'segs': stream['segs']
                     })
 
@@ -252,14 +252,14 @@ def prepare(self, **kwargs):
                         'video_profile': stream_types[stream_id]['video_profile'],
                         'size': stream['size'],
                         'pieces': [{
-                            'fileid': stream['stream_fileid'],
+                            'fileid': stream['segs'][0]['fileid'],
                             'segs': stream['segs']
                         }]
                     }
                 else:
                     self.streams_fallback[stream_id]['size'] += stream['size']
                     self.streams_fallback[stream_id]['pieces'].append({
-                        'fileid': stream['stream_fileid'],
+                        'fileid': stream['segs'][0]['fileid'],
                         'segs': stream['segs']
                     })
 

From 15a1a20fb9e4a92f10dcbf8494ad3555ac4d4e61 Mon Sep 17 00:00:00 2001
From: Adam Cavendish <GetbetterABC@yeah.net>
Date: Sun, 16 Apr 2017 00:12:54 +0800
Subject: [PATCH 0351/1225] Cleaner fix for the last commit

---
 src/you_get/extractors/youku.py | 18 +++++-------------
 1 file changed, 5 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index f36d633947..c3ad67eb0e 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -53,18 +53,14 @@ def trans_e(a, c):
 
         return result
 
-    def generate_ep(self, no, streamfileids, sid, token):
-        number = hex(int(str(no), 10))[2:].upper()
-        if len(number) == 1:
-            number = '0' + number
-        fileid = streamfileids[0:8] + number + streamfileids[10:]
+    def generate_ep(self, fileid, sid, token):
         ep = parse.quote(base64.b64encode(
             ''.join(self.__class__.trans_e(
                 self.f_code_2,  #use the 86 fcode if using 86
                 sid + '_' + fileid + '_' + token)).encode('latin1')),
             safe='~()*!.\''
         )
-        return fileid, ep
+        return ep
 
     # Obsolete -- used to parse m3u8 on pl.youku.com
     def parse_m3u8(m3u8):
@@ -228,14 +224,12 @@ def prepare(self, **kwargs):
                         'video_profile': stream_types[stream_id]['video_profile'],
                         'size': stream['size'],
                         'pieces': [{
-                            'fileid': stream['segs'][0]['fileid'],
                             'segs': stream['segs']
                         }]
                     }
                 else:
                     self.streams[stream_id]['size'] += stream['size']
                     self.streams[stream_id]['pieces'].append({
-                        'fileid': stream['segs'][0]['fileid'],
                         'segs': stream['segs']
                     })
 
@@ -252,14 +246,12 @@ def prepare(self, **kwargs):
                         'video_profile': stream_types[stream_id]['video_profile'],
                         'size': stream['size'],
                         'pieces': [{
-                            'fileid': stream['segs'][0]['fileid'],
                             'segs': stream['segs']
                         }]
                     }
                 else:
                     self.streams_fallback[stream_id]['size'] += stream['size']
                     self.streams_fallback[stream_id]['pieces'].append({
-                        'fileid': stream['segs'][0]['fileid'],
                         'segs': stream['segs']
                     })
 
@@ -294,17 +286,17 @@ def extract(self, **kwargs):
                 pieces = self.streams[stream_id]['pieces']
                 for piece in pieces:
                     segs = piece['segs']
-                    streamfileid = piece['fileid']
                     seg_count = len(segs)
                     for no in range(0, seg_count):
                         k = segs[no]['key']
+                        fileid = segs[no]['fileid']
                         if k == -1:
                             # we hit the paywall; stop here
                             log.w('Skipping %d out of %d segments due to paywall' %
                                   (seg_count - no, seg_count))
                             break
-                        fileid, ep = self.__class__.generate_ep(self, no, streamfileid,
-                                                                sid, token)
+                        ep = self.__class__.generate_ep(self, fileid,
+                                                        sid, token)
                         q = parse.urlencode(dict(
                             ctype = self.ctype,
                             ev    = 1,

From f65fe485cd141bd4d68121e1586e789109a6bc5c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 16 Apr 2017 12:25:19 +0200
Subject: [PATCH 0352/1225] version 0.4.715

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 63d908c602..fd8d7bc010 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.652'
+__version__ = '0.4.715'

From 45a3beaffe3768f2c249dc5ba4c1ae8faaecc750 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 25 Feb 2017 01:57:33 +0800
Subject: [PATCH 0353/1225] fix tudou.py when there is no title

---
 src/you_get/extractors/tudou.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index 8c434437c7..ebf3512397 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -26,7 +26,10 @@ def tudou_download_by_id(id, title, output_dir = '.', merge = True, info_only =
     html = get_html('http://www.tudou.com/programs/view/%s/' % id)
 
     iid = r1(r'iid\s*[:=]\s*(\S+)', html)
-    title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
+    try:
+        title = r1(r'kw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
+    except AttributeError:
+        title = ''
     tudou_download_by_iid(iid, title, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
@@ -44,9 +47,12 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwa
 
     html = get_decoded_html(url)
 
-    title = r1(r'\Wkw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
-    assert title
-    title = unescape_html(title)
+    try:
+        title = r1(r'\Wkw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
+        assert title
+        title = unescape_html(title)
+    except AttributeError:
+        title = ''
 
     vcode = r1(r'vcode\s*[:=]\s*\'([^\']+)\'', html)
     if vcode:

From ccb1bb9c7762713fa3983da3d00d73a19c08d56e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 20 Apr 2017 17:24:30 +0800
Subject: [PATCH 0354/1225] new matching patterns

---
 src/you_get/extractors/tudou.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index ebf3512397..ad37a191c2 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -45,16 +45,20 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     if id:
         return tudou_download_by_id(id, title="", info_only=info_only)
 
-    html = get_decoded_html(url)
+    html = get_content(url)
 
     try:
         title = r1(r'\Wkw\s*[:=]\s*[\'\"]([^\n]+?)\'\s*\n', html).replace("\\'", "\'")
         assert title
         title = unescape_html(title)
     except AttributeError:
-        title = ''
+        title = match1(html, r'id=\"subtitle\"\s*title\s*=\s*\"([^\"]+)\"')
+        if title is None:
+            title = ''
 
     vcode = r1(r'vcode\s*[:=]\s*\'([^\']+)\'', html)
+    if vcode is None:
+        vcode = match1(html, r'viden\s*[:=]\s*\"([\w+/=]+)\"')
     if vcode:
         from .youku import youku_download_by_vid
         return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)

From bdffcfacab5069019c684b61562a46cf9e87a99d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 21 Apr 2017 00:49:00 +0800
Subject: [PATCH 0355/1225] tuning debugging message

---
 src/you_get/common.py           | 5 +----
 src/you_get/processor/ffmpeg.py | 4 +++-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 29900c0bf1..5a1ff07fb0 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -279,15 +279,11 @@ def get_response(url, faker = False):
 
 # DEPRECATED in favor of get_content()
 def get_html(url, encoding = None, faker = False):
-    logging.debug('get_html: %s' % url)
-
     content = get_response(url, faker).data
     return str(content, 'utf-8', 'ignore')
 
 # DEPRECATED in favor of get_content()
 def get_decoded_html(url, faker = False):
-    logging.debug('get_decoded_html: %s' % url)
-
     response = get_response(url, faker)
     data = response.data
     charset = r1(r'charset=([\w-]+)', response.headers['content-type'])
@@ -1362,6 +1358,7 @@ def getaddrinfo(*args):
         else:
             sys.exit(1)
     except UnicodeEncodeError:
+        raise
         log.e('[error] oops, the current environment does not seem to support Unicode.')
         log.e('please set it to a UTF-8-aware locale first,')
         log.e('so as to save the video (with some Unicode characters) correctly.')
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 78cd83fa13..c7631b600c 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -21,10 +21,12 @@ def get_usable_ffmpeg(cmd):
         out, err = p.communicate()
         vers = str(out, 'utf-8').split('\n')[0].split()
         assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
-        #if the version is strange like 'N-1234-gd1111', set version to 2.0
+        #set version to 1.0 for nightly build and print warning
         try:
             version = [int(i) for i in vers[2].split('.')]
         except:
+            print('It seems that your ffmpeg is a nightly build.')
+            print('Please switch to the latest stable if merging failed.')
             version = [1, 0]
         return cmd, version
     except:

From fd661668a78fbb5fc3fba4b6aca9d91a23e01129 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 21 Apr 2017 01:01:13 +0800
Subject: [PATCH 0356/1225] print stack trace only when debug flag is on

---
 src/you_get/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5a1ff07fb0..49de21a1d9 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1358,7 +1358,8 @@ def getaddrinfo(*args):
         else:
             sys.exit(1)
     except UnicodeEncodeError:
-        raise
+        if traceback:
+            raise
         log.e('[error] oops, the current environment does not seem to support Unicode.')
         log.e('please set it to a UTF-8-aware locale first,')
         log.e('so as to save the video (with some Unicode characters) correctly.')

From 63e6c4da416181af578f2cb93c100b2035276e0b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 24 Apr 2017 13:54:23 +0800
Subject: [PATCH 0357/1225] fix iqiyi tiltes when accessed with curid=

---
 src/you_get/extractors/iqiyi.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 91329708a2..a13718e320 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -134,7 +134,12 @@ def prepare(self, **kwargs):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html)
             self.vid = (tvid, videoid)
-            self.title = match1(html, '<title>([^<]+)').split('-')[0]
+            info_u = 'http://mixer.video.iqiyi.com/jp/mixin/videos/' + tvid
+            mixin = get_content(info_u)
+            mixin_json = json.loads(mixin[len('var tvInfoJs='):])
+            real_u = mixin_json['url']
+            real_html = get_content(real_u)
+            self.title = match1(real_html, '<title>([^<]+)').split('-')[0]
         tvid, videoid = self.vid
         info = getVMS(tvid, videoid)
         assert info['code'] == 'A00000', 'can\'t play this video'

From 2425edc5e738493830dd5ad91e341c7177867ead Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 24 Apr 2017 20:37:45 +0800
Subject: [PATCH 0358/1225] refactoring

---
 src/you_get/extractors/miaopai.py | 54 ++++++++++++++-----------------
 1 file changed, 25 insertions(+), 29 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 72dfdaf705..e9a6ff2df2 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -5,39 +5,35 @@
 from ..common import *
 import urllib.error
 
-def miaopai_download_by_url(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    if re.match(r'http://video.weibo.com/show\?fid=(\d{4}:\w{32})\w*', url):
-        fake_headers_mobile = {
-            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-            'Accept-Charset': 'UTF-8,*;q=0.5',
-            'Accept-Encoding': 'gzip,deflate,sdch',
-            'Accept-Language': 'en-US,en;q=0.8',
-            'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
-        }
-        webpage_url = re.search(r'(http://video.weibo.com/show\?fid=\d{4}:\w{32})\w*', url).group(1) + '&type=mp4'  #mobile
-
-        #grab download URL
-        a = get_content(webpage_url, headers= fake_headers_mobile , decoded=True)
-        url = match1(a, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
-
-        #grab title
-        b = get_content(webpage_url)  #normal
-        title = match1(b, r'<meta name="description" content="([\s\S]*?)\"\W')
-
-        type_, ext, size = url_info(url)
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([url], title.replace('\n',''), ext, total_size=None, output_dir=output_dir, merge=merge)
+    fake_headers_mobile = {
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Charset': 'UTF-8,*;q=0.5',
+        'Accept-Encoding': 'gzip,deflate,sdch',
+        'Accept-Language': 'en-US,en;q=0.8',
+        'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
+    }
+    page_url = 'http://video.weibo.com/show?fid=' + fid + '&type=mp4'
+
+    mobile_page = get_content(page_url, headers=fake_headers_mobile)
+    url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
+    title = match1(mobile_page, r'<title>([^<]+)</title>')
+    type_, ext, size = url_info(url)
+    print_info(site_info, title, type_, size)
+    if not info_only:
+        download_urls([url], title.replace('\n',''), ext, total_size=None, output_dir=output_dir, merge=merge)
 
 #----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    """"""
-    if re.match(r'http://video.weibo.com/show\?fid=(\d{4}:\w{32})\w*', url):
-        miaopai_download_by_url(url, output_dir, merge, info_only)
-    elif re.match(r'http://weibo.com/p/230444\w+', url):
-        _fid = match1(url, r'http://weibo.com/p/230444(\w+)')
-        miaopai_download_by_url('http://video.weibo.com/show?fid=1034:{_fid}'.format(_fid = _fid), output_dir, merge, info_only)
+    fid = match1(url, r'\?fid=(\d{4}:\w{32})')
+    if fid is not None:
+        miaopai_download_by_fid(fid, output_dir, merge, info_only)
+    elif '/p/230444' in url:
+        fid = match1(url, r'/p/230444(\w+)')
+        miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
+    else:
+        raise Exception('Unknown pattern')
 
 site_info = "miaopai"
 download = miaopai_download

From 34d05f17779ae7a45d1cda23729bc85c668768b3 Mon Sep 17 00:00:00 2001
From: ultimate010 <ultimate010@gmail.com>
Date: Tue, 25 Apr 2017 16:58:36 +0800
Subject: [PATCH 0359/1225] add suport for toutiao.com

---
 src/you_get/common.py             |   1 +
 src/you_get/extractors/toutiao.py | 134 ++++++++++++++++++++++++++++++
 2 files changed, 135 insertions(+)
 create mode 100644 src/you_get/extractors/toutiao.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 49de21a1d9..54819741be 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -94,6 +94,7 @@
     'youtu'            : 'youtube',
     'youtube'          : 'youtube',
     'zhanqi'           : 'zhanqi',
+    '365yg': 'toutiao',
 }
 
 import getopt
diff --git a/src/you_get/extractors/toutiao.py b/src/you_get/extractors/toutiao.py
new file mode 100644
index 0000000000..ebd3013f01
--- /dev/null
+++ b/src/you_get/extractors/toutiao.py
@@ -0,0 +1,134 @@
+#!/usr/bin/env python
+import base64
+
+from ..common import *
+import random
+from json import loads
+
+__all__ = ['toutiao_download', ]
+
+
+# magic function
+def int_overflow(val):
+    maxint = 2147483647
+    if not -maxint - 1 <= val <= maxint:
+        val = (val + (maxint + 1)) % (2 * (maxint + 1)) - maxint - 1
+    return val
+
+import ctypes
+
+
+def unsigned_right_shitf(n, i):
+    if n < 0:
+        n = ctypes.c_uint32(n).value
+    if i < 0:
+        return -int_overflow(n << abs(i))
+    return int_overflow(n >> i)
+
+
+def gen_table():
+    t = [0] * 256
+    for r in range(256):
+        e = r
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        e = (-306674912 ^ unsigned_right_shitf(e, 1)
+             ) if 1 & e else unsigned_right_shitf(e, 1)
+        t[r] = e
+    return t
+
+table = gen_table()
+
+
+def sign_url(r, url):
+    a = len(url)
+    t = -1
+    n = -1
+    o = -1
+    for i in range(a):
+        t = ord(url[i])
+        if t < 128:
+            o = unsigned_right_shitf(o, 8) ^ r[255 & (o ^ t)]
+    return o ^ -1
+
+
+def sign_video_url(vid):
+    href = "http://i.snssdk.com/video/urls/v/1/toutiao/mp4/" + vid
+    o = "/video/urls/v/1/toutiao/mp4/" + vid + "?r=" + \
+        str(random.randint(10000000000000000, 999999999999999999))
+    t = sign_url(table, o)
+    i = 4294967296 + t if t < 0 else t
+    return "http:" + "//" + "i.snssdk.com" + o + "&s=" + str(i)
+
+
+class ToutiaoVideoInfo(object):
+
+    def __init__(self):
+        self.bitrate = None
+        self.definition = None
+        self.size = None
+        self.height = None
+        self.width = None
+        self.type = None
+        self.url = None
+
+    def __str__(self):
+        return json.dumps(self.__dict__)
+
+
+def get_file_by_vid(video_id):
+    vRet = []
+    url = sign_video_url(video_id)
+    ret = get_content(url)
+    ret = loads(ret)
+    vlist = ret.get('data').get('video_list')
+    if len(vlist) > 0:
+        vInfo = vlist.get(sorted(vlist.keys(), reverse=True)[0])
+        vUrl = vInfo.get('main_url')
+        vUrl = base64.decodestring(vUrl.encode('ascii')).decode('ascii')
+        videoInfo = ToutiaoVideoInfo()
+        videoInfo.bitrate = vInfo.get('bitrate')
+        videoInfo.definition = vInfo.get('definition')
+        videoInfo.size = vInfo.get('size')
+        videoInfo.height = vInfo.get('vheight')
+        videoInfo.width = vInfo.get('vwidth')
+        videoInfo.type = vInfo.get('vtype')
+        videoInfo.url = vUrl
+        vRet.append(videoInfo)
+    return vRet
+
+
+def toutiao_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    html = get_html(url, faker=True)
+    video_id = match1(html, r"videoid\s*:\s*'([^']+)',\n")
+    title = match1(html, r"title: '([^']+)'.replace")
+    video_file_list = get_file_by_vid(video_id)  # 调api获取视频源文件
+    type, ext, size = url_info(video_file_list[0].url, faker=True)
+    log.d(video_file_list[0].url)
+    print_info(site_info=site_info, title=title, type=type, size=size)
+    if not info_only:
+        download_urls(
+            [video_file_list[0].url],
+            title,
+            ext,
+            size,
+            output_dir,
+            merge=merge,
+            faker=True)
+
+
+site_info = "Toutiao.com"
+download = toutiao_download
+download_playlist = playlist_not_supported("toutiao")

From 3246079751d39b3e898b627445c7df59e7b631a9 Mon Sep 17 00:00:00 2001
From: ultimate010 <ultimate010@gmail.com>
Date: Wed, 26 Apr 2017 09:42:39 +0800
Subject: [PATCH 0360/1225] use crc32 to sign

Signed-off-by: ultimate010 <ultimate010@gmail.com>
---
 src/you_get/extractors/toutiao.py | 83 +++++--------------------------
 1 file changed, 13 insertions(+), 70 deletions(-)

diff --git a/src/you_get/extractors/toutiao.py b/src/you_get/extractors/toutiao.py
index ebd3013f01..03f7a13b6a 100644
--- a/src/you_get/extractors/toutiao.py
+++ b/src/you_get/extractors/toutiao.py
@@ -1,6 +1,8 @@
 #!/usr/bin/env python
 import base64
 
+import binascii
+
 from ..common import *
 import random
 from json import loads
@@ -8,69 +10,18 @@
 __all__ = ['toutiao_download', ]
 
 
-# magic function
-def int_overflow(val):
-    maxint = 2147483647
-    if not -maxint - 1 <= val <= maxint:
-        val = (val + (maxint + 1)) % (2 * (maxint + 1)) - maxint - 1
-    return val
-
-import ctypes
-
-
-def unsigned_right_shitf(n, i):
-    if n < 0:
-        n = ctypes.c_uint32(n).value
-    if i < 0:
-        return -int_overflow(n << abs(i))
-    return int_overflow(n >> i)
-
-
-def gen_table():
-    t = [0] * 256
-    for r in range(256):
-        e = r
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        e = (-306674912 ^ unsigned_right_shitf(e, 1)
-             ) if 1 & e else unsigned_right_shitf(e, 1)
-        t[r] = e
-    return t
-
-table = gen_table()
-
-
-def sign_url(r, url):
-    a = len(url)
-    t = -1
-    n = -1
-    o = -1
-    for i in range(a):
-        t = ord(url[i])
-        if t < 128:
-            o = unsigned_right_shitf(o, 8) ^ r[255 & (o ^ t)]
-    return o ^ -1
+def sign_video_url(vid):
+    # some code from http://codecloud.net/110854.html
+    r = str(random.random())[2:]
 
+    def right_shift(val, n):
+        return val >> n if val >= 0 else (val + 0x100000000) >> n
 
-def sign_video_url(vid):
-    href = "http://i.snssdk.com/video/urls/v/1/toutiao/mp4/" + vid
-    o = "/video/urls/v/1/toutiao/mp4/" + vid + "?r=" + \
-        str(random.randint(10000000000000000, 999999999999999999))
-    t = sign_url(table, o)
-    i = 4294967296 + t if t < 0 else t
-    return "http:" + "//" + "i.snssdk.com" + o + "&s=" + str(i)
+    url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % vid
+    n = url.replace("http://i.snssdk.com", "")+ '?r=' + r
+    c = binascii.crc32(n.encode("ascii"))
+    s = right_shift(c, 0)
+    return url + '?r=%s&s=%s' % (r, s)
 
 
 class ToutiaoVideoInfo(object):
@@ -116,17 +67,9 @@ def toutiao_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     title = match1(html, r"title: '([^']+)'.replace")
     video_file_list = get_file_by_vid(video_id)  # 调api获取视频源文件
     type, ext, size = url_info(video_file_list[0].url, faker=True)
-    log.d(video_file_list[0].url)
     print_info(site_info=site_info, title=title, type=type, size=size)
     if not info_only:
-        download_urls(
-            [video_file_list[0].url],
-            title,
-            ext,
-            size,
-            output_dir,
-            merge=merge,
-            faker=True)
+        download_urls([video_file_list[0].url], title, ext, size, output_dir, merge=merge, faker=True)
 
 
 site_info = "Toutiao.com"

From 398845fd69479f5ffbf942b46004ec54f9453fcf Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 26 Apr 2017 11:15:04 +0800
Subject: [PATCH 0361/1225] [qq]fix v.qq.com/page redirect

---
 src/you_get/extractors/qq.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 08d889b61a..4a9d5f898d 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -134,8 +134,13 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if 'v.qq.com/page' in url:
         # for URLs like this:
         # http://v.qq.com/page/k/9/7/k0194pwgw97.html
-        content = get_html(url)
-        url = match1(content,r'window\.location\.href="(.*?)"')
+        new_url = url_locations([url])[0]
+        if url == new_url:
+            #redirect in js?
+            content = get_content(url)
+            url = match1(content,r'window\.location\.href="(.*?)"')
+        else:
+            url = new_url
 
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_html(url)

From 5cf63526c6c23ed08619ebe47085440ebdff6b9b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 4 May 2017 11:34:18 +0800
Subject: [PATCH 0362/1225] [le.py] update api

---
 src/you_get/extractors/le.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index 9de6c7a7fe..bc034f3982 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -26,7 +26,9 @@ def get_key(t):
 
 def calcTimeKey(t):
     ror = lambda val, r_bits, : ((val & (2**32-1)) >> r_bits%32) |  (val << (32-(r_bits%32)) & (2**32-1))
-    return ror(ror(t,773625421%13)^773625421,773625421%17)
+    magic = 185025305
+    return ror(t, magic % 17) ^ magic
+    #return ror(ror(t,773625421%13)^773625421,773625421%17)
 
 
 def decode(data):
@@ -52,9 +54,10 @@ def decode(data):
 
 
 def video_info(vid,**kwargs):
-    url = 'http://api.letv.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.letv.com'.format(vid,calcTimeKey(int(time.time())))
+    url = 'http://player-pc.le.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.le.com&region=cn&source=1000&accesyx=1'.format(vid,calcTimeKey(int(time.time())))
     r = get_content(url, decoded=False)
     info=json.loads(str(r,"utf-8"))
+    info = info['msgs']
 
 
     stream_id = None
@@ -73,15 +76,18 @@ def video_info(vid,**kwargs):
             stream_id =sorted(support_stream_id,key= lambda i: int(i[1:]))[-1]
 
     url =info["playurl"]["domain"][0]+info["playurl"]["dispatch"][stream_id][0]
+    uuid = hashlib.sha1(url.encode('utf8')).hexdigest() + '_0'
     ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1]
-    url+="&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn={}&pay=0&iscpn=f9051&rateid={}".format(random.random(),stream_id)
+    url = url.replace('tss=0', 'tss=ios')
+    url+="&m3v=1&termid=1&format=1&hwtype=un&ostype=MacOS10.12.4&p1=1&p2=10&p3=-&expect=3&tn={}&vid={}&uuid={}&sign=letv".format(random.random(), vid, uuid)
 
     r2=get_content(url,decoded=False)
     info2=json.loads(str(r2,"utf-8"))
 
     # hold on ! more things to do
     # to decode m3u8 (encoded)
-    m3u8 = get_content(info2["location"],decoded=False)
+    suffix = '&r=' + str(int(time.time() * 1000)) + '&appid=500'
+    m3u8 = get_content(info2["location"]+suffix,decoded=False)
     m3u8_list = decode(m3u8)
     urls = re.findall(r'^[^#][^\r]*',m3u8_list,re.MULTILINE)
     return ext,urls

From 27b80072c38bb560ecda7050091436b46641278f Mon Sep 17 00:00:00 2001
From: kaka <kaka.chen2009@gmail.com>
Date: Sat, 6 May 2017 20:11:21 +0800
Subject: [PATCH 0363/1225] strip name of album fixed when download some album
 which name endwith spaces raise FileNotFoundError problem.

---
 src/you_get/extractors/netease.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index 17ae70a930..f74747b189 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -29,7 +29,7 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
         j = loads(get_content("http://music.163.com/api/album/%s?id=%s&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
 
         artist_name = j['album']['artists'][0]['name']
-        album_name = j['album']['name']
+        album_name = j['album']['name'].strip()
         new_dir = output_dir + '/' + fs.legitimize("%s - %s" % (artist_name, album_name))
         if not info_only:
             if not os.path.exists(new_dir):

From 5777d49f62df89018c38113bf6df55970ea46988 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 6 May 2017 15:11:50 +0200
Subject: [PATCH 0364/1225] [bilibili] add referer

---
 src/you_get/extractors/bilibili.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a776f9a612..ccb395cb56 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -82,7 +82,7 @@ def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only
 
     print_info(site_info, title, type_, size)
     if not info_only:
-        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge)
+        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, headers={'Referer': 'http://www.bilibili.com/'})
 
 
 def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
@@ -98,12 +98,12 @@ def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=F
             type_ = ''
             size = 0
             for url in urls:
-                _, type_, temp = url_info(url)
+                _, type_, temp = url_info(url, headers={'Referer': 'http://www.bilibili.com/'})
                 size += temp or 0
 
             print_info(site_info, title, type_, size)
             if not info_only:
-                download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, timeout=1)
+                download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, timeout=1, headers={'Referer': 'http://www.bilibili.com/'})
         except socket.timeout:
             continue
         else:

From f64b3494c71e2388ac7c37a7549a7e5e82082131 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 6 May 2017 15:31:29 +0200
Subject: [PATCH 0365/1225] [common] url_save: always specify Range

---
 src/you_get/common.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 49de21a1d9..2bdc365527 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -518,8 +518,7 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
             headers = headers
         else:
             headers = {}
-        if received:
-            headers['Range'] = 'bytes=' + str(received) + '-'
+        headers['Range'] = 'bytes=' + str(received) + '-'
         if refer:
             headers['Referer'] = refer
 

From 8f828a90edf0f9c2d91e19a23d99fe41d414b700 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 7 May 2017 00:47:06 +0800
Subject: [PATCH 0366/1225] [common] url_info should not ignore refer when
 called in url_save

---
 src/you_get/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2bdc365527..90cb864bd5 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -479,6 +479,9 @@ def url_locations(urls, faker = False, headers = {}):
     return locations
 
 def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}, timeout = None, **kwargs):
+#When a referer specified with param refer, the key must be 'Referer' for the hack here
+    if refer is not None:
+        headers['Referer'] = refer
     file_size = url_size(url, faker = faker, headers = headers)
 
     if os.path.exists(filepath):

From 65712f8619502a452938320cc4d7151e1c3c8206 Mon Sep 17 00:00:00 2001
From: XiaochenCui <jcnlcxc@163.com>
Date: Sun, 7 May 2017 07:20:00 +0800
Subject: [PATCH 0367/1225] [ku6]fix error when parse url's extension

---
 src/you_get/extractors/ku6.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index 808619f439..9996d7e4d1 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -14,7 +14,7 @@ def ku6_download_by_id(id, title = None, output_dir = '.', merge = True, info_on
     title = title or t
     assert title
     urls = f.split(',')
-    ext = re.sub(r'.*\.', '', urls[0])
+    ext = match1(urls[0], r'.*\.(\w+)\??[^\.]*')
     assert ext in ('flv', 'mp4', 'f4v'), ext
     ext = {'f4v': 'flv'}.get(ext, ext)
     size = 0

From 3bd8910ffb1a85a3f70ddb312dbff9e9632f23fc Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 9 May 2017 10:17:42 +0800
Subject: [PATCH 0368/1225] [ximalaya] use fake_headers to fetch json

---
 src/you_get/extractors/ximalaya.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ximalaya.py b/src/you_get/extractors/ximalaya.py
index 58a158ba57..58e2945cb2 100644
--- a/src/you_get/extractors/ximalaya.py
+++ b/src/you_get/extractors/ximalaya.py
@@ -15,7 +15,8 @@
 
 def ximalaya_download_by_id(id, title = None, output_dir = '.', info_only = False, stream_id = None):
     BASE_URL = 'http://www.ximalaya.com/tracks/'
-    json_data = json.loads(get_content(BASE_URL + id + '.json'))
+    json_url = BASE_URL + id + '.json'
+    json_data = json.loads(get_content(json_url, headers=fake_headers))
     if 'res' in json_data:
         if json_data['res'] == False:
             raise ValueError('Server reported id %s is invalid' % id)

From d3d9e0f5c0da8408bcdb241509cb7dd1f41fd4bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 13 May 2017 15:00:31 +0200
Subject: [PATCH 0369/1225] [imgur] use hash when title not present

---
 src/you_get/extractors/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index b7bdc4c5bd..cc5dc9fd88 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -65,7 +65,7 @@ def prepare(self, **kwargs):
                     'container': 'jpg'
                 }
             }
-            self.title = image['title']
+            self.title = image['title'] or image['hash']
 
     def extract(self, **kwargs):
         if 'stream_id' in kwargs and kwargs['stream_id']:

From 08fdd2246693ce49891f046e2b5840b08397e82a Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sat, 13 May 2017 17:15:18 -0400
Subject: [PATCH 0370/1225] [common] add .ac.cn support

---
 src/you_get/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 90cb864bd5..8d4d2d7683 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -75,6 +75,7 @@
     'tumblr'           : 'tumblr',
     'twimg'            : 'twitter',
     'twitter'          : 'twitter',
+    'ucas'             : 'ucas',
     'videomega'        : 'videomega',
     'vidto'            : 'vidto',
     'vimeo'            : 'vimeo',
@@ -1412,7 +1413,7 @@ def url_to_module(url):
         video_host = r1(r'https?://([^/]+)/', url)
         video_url = r1(r'https?://[^/]+(.*)', url)
 
-    if video_host.endswith('.com.cn'):
+    if video_host.endswith('.com.cn') or video_host.endswith('.ac.cn'):
         video_host = video_host[:-3]
     domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
     assert domain, 'unsupported url: ' + url

From d0431d8bd6bdf59759e021526dc28152441e5c1a Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sat, 13 May 2017 17:15:29 -0400
Subject: [PATCH 0371/1225] [UCAS] add support

---
 src/you_get/extractors/__init__.py |   1 +
 src/you_get/extractors/ucas.py     | 136 +++++++++++++++++++++++++++++
 2 files changed, 137 insertions(+)
 create mode 100644 src/you_get/extractors/ucas.py

diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index a027c396d5..6d6f56cf94 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -66,6 +66,7 @@
 from .tudou import *
 from .tumblr import *
 from .twitter import *
+from .ucas import *
 from .veoh import *
 from .videomega import *
 from .vimeo import *
diff --git a/src/you_get/extractors/ucas.py b/src/you_get/extractors/ucas.py
new file mode 100644
index 0000000000..735a80cd38
--- /dev/null
+++ b/src/you_get/extractors/ucas.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+
+__all__ = ['ucas_download', 'ucas_download_single', 'ucas_download_playlist']
+
+from ..common import *
+import urllib.error
+import requests
+from time import time
+from random import random
+import xml.etree.ElementTree as ET
+from copy import copy
+
+"""
+Do not replace request.get with get_content
+for UCAS's server is not correctly returning data!
+"""
+
+def dictify(r,root=True):
+    """http://stackoverflow.com/a/30923963/2946714"""
+    if root:
+        return {r.tag : dictify(r, False)}
+    d=copy(r.attrib)
+    if r.text:
+        d["_text"]=r.text
+    for x in r.findall("./*"):
+        if x.tag not in d:
+            d[x.tag]=[]
+        d[x.tag].append(dictify(x,False))
+    return d
+
+def _get_video_query_url(resourceID):
+    # has to be like this
+    headers = {
+        'DNT': '1',
+        'Accept-Encoding': 'gzip, deflate',
+        'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.47 Safari/537.36',
+        'Accept': '*/*',
+        'Referer': 'http://v.ucas.ac.cn/',
+        'Connection': 'keep-alive',
+    }
+    params = (
+        ('method', 'query'),
+        ('loginname', 'videocas'),
+        ('pwd', 'af1c7a4c5f77f790722f7cae474c37e281203765d423a23b'),
+        ('resource', '[{"resourceID":"' + resourceID + '","on":1,"time":60000,"eid":100,"w":800,"h":600}]'),
+        ('timeStamp', '{timeStamp}'.format(timeStamp = int(time()))),
+    )
+    a = requests.get('http://210.76.211.10/vplus/remote.do', headers=headers, params=params)
+    info =  a.content.decode('utf-8')
+    return match1(info, r'video":"(.+)"')
+
+def _get_virtualPath(video_query_url):
+    #getResourceJsCode2
+    html = requests.get(video_query_url)
+    html =  html.content.decode('utf-8')
+    
+    return match1(html, r"function\s+getVirtualPath\(\)\s+{\s+return\s+'(\w+)'")
+
+
+
+def _get_video_list(resourceID):
+    """"""
+    video_xml = requests.get('http://210.76.211.10/vplus/member/resource.do?isyulan=0&method=queryFlashXmlByResourceId&resourceId={resourceID}&randoms={randoms}'.format(resourceID = resourceID,
+                                                                                                                                                                         randoms = random()))    
+    video_xml = video_xml.content.decode('utf-8')
+
+    root = ET.fromstring(video_xml.split('___!!!___')[0])
+
+    r = dictify(root)
+
+    huge_list = []
+    # main
+    huge_list.append([i['value'] for i in sorted(r['video']['mainUrl'][0]['_flv'][0]['part'][0]['video'], key=lambda k: int(k['index']))])
+
+    # sub
+    if '_flv' in r['video']['subUrl'][0]:
+        huge_list.append([i['value'] for i in sorted(r['video']['subUrl'][0]['_flv'][0]['part'][0]['video'], key=lambda k: int(k['index']))])
+
+    return huge_list
+
+def _ucas_get_url_lists_by_resourceID(resourceID):
+    video_query_url = _get_video_query_url(resourceID)
+    assert video_query_url != '', 'Cannot find video GUID!'
+    
+    virtualPath = _get_virtualPath(video_query_url)
+    assert virtualPath != '', 'Cannot find virtualPath!'
+    
+    url_lists = _get_video_list(resourceID)
+    assert url_lists, 'Cannot find any URL to download!'
+
+    # make real url
+    # credit to a mate in UCAS
+    for video_type_id, video_urls in enumerate(url_lists):
+        for k, path in enumerate(video_urls):
+            url_lists[video_type_id][k] = 'http://210.76.211.10/vplus/member/resource.do?virtualPath={virtualPath}&method=getImgByStream&imgPath={path}'.format(virtualPath = virtualPath,
+                                                                                                                                                                path = path)
+
+    return url_lists
+
+def ucas_download_single(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    '''video page'''
+    html = get_content(url)
+    # resourceID is UUID
+    resourceID = re.findall( r'resourceID":"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', html)[0]
+    assert resourceID != '', 'Cannot find resourceID!'
+
+    title = match1(html, r'<div class="bc-h">(.+)</div>')
+    url_lists = _ucas_get_url_lists_by_resourceID(resourceID)
+    assert url_lists, 'Cannot find any URL of such class!'
+    
+    for k, part in enumerate(url_lists):
+        part_title = title + '_' + str(k)
+        print_info(site_info, part_title, 'flv', 0)
+        if not info_only:
+            download_urls(part, part_title, 'flv', total_size=None, output_dir=output_dir, merge=merge)
+
+def ucas_download_playlist(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    '''course page'''
+    html = get_content(url)
+
+    parts = re.findall( r'(getplaytitle.do\?.+)"', html)
+    assert parts, 'No part found!'
+
+    for part_path in parts:
+        ucas_download('http://v.ucas.ac.cn/course/' + part_path, output_dir=output_dir, merge=merge, info_only=info_only)
+
+def ucas_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    if 'classid=' in url and 'getplaytitle.do' in url:
+        ucas_download_single(url, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif 'CourseIndex.do' in url:
+        ucas_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
+
+site_info = "UCAS"
+download = ucas_download
+download_playlist = ucas_download_playlist
\ No newline at end of file

From c7a262cf652ec51485634fb92d58a32d2d63504f Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Sat, 13 May 2017 17:39:07 -0400
Subject: [PATCH 0372/1225] [UCAS] use http.client instead of requests

---
 src/you_get/extractors/ucas.py | 35 +++++++++++++++++-----------------
 1 file changed, 18 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/ucas.py b/src/you_get/extractors/ucas.py
index 735a80cd38..4f07bce5a0 100644
--- a/src/you_get/extractors/ucas.py
+++ b/src/you_get/extractors/ucas.py
@@ -4,14 +4,14 @@
 
 from ..common import *
 import urllib.error
-import requests
+import http.client
 from time import time
 from random import random
 import xml.etree.ElementTree as ET
 from copy import copy
 
 """
-Do not replace request.get with get_content
+Do not replace http.client with get_content
 for UCAS's server is not correctly returning data!
 """
 
@@ -39,31 +39,32 @@ def _get_video_query_url(resourceID):
         'Referer': 'http://v.ucas.ac.cn/',
         'Connection': 'keep-alive',
     }
-    params = (
-        ('method', 'query'),
-        ('loginname', 'videocas'),
-        ('pwd', 'af1c7a4c5f77f790722f7cae474c37e281203765d423a23b'),
-        ('resource', '[{"resourceID":"' + resourceID + '","on":1,"time":60000,"eid":100,"w":800,"h":600}]'),
-        ('timeStamp', '{timeStamp}'.format(timeStamp = int(time()))),
-    )
-    a = requests.get('http://210.76.211.10/vplus/remote.do', headers=headers, params=params)
-    info =  a.content.decode('utf-8')
+    conn = http.client.HTTPConnection("210.76.211.10")
+    
+    conn.request("GET", "/vplus/remote.do?method=query2&loginname=videocas&pwd=af1c7a4c5f77f790722f7cae474c37e281203765d423a23b&resource=%5B%7B%22resourceID%22%3A%22" + resourceID + "%22%2C%22on%22%3A1%2C%22time%22%3A600%2C%22eid%22%3A100%2C%22w%22%3A800%2C%22h%22%3A600%7D%5D&timeStamp=" + str(int(time())), headers=headers)
+    res = conn.getresponse()
+    data = res.read()
+
+    info =  data.decode("utf-8")
     return match1(info, r'video":"(.+)"')
 
 def _get_virtualPath(video_query_url):
     #getResourceJsCode2
-    html = requests.get(video_query_url)
-    html =  html.content.decode('utf-8')
+    html = get_content(video_query_url)
     
     return match1(html, r"function\s+getVirtualPath\(\)\s+{\s+return\s+'(\w+)'")
 
 
-
 def _get_video_list(resourceID):
     """"""
-    video_xml = requests.get('http://210.76.211.10/vplus/member/resource.do?isyulan=0&method=queryFlashXmlByResourceId&resourceId={resourceID}&randoms={randoms}'.format(resourceID = resourceID,
-                                                                                                                                                                         randoms = random()))    
-    video_xml = video_xml.content.decode('utf-8')
+    conn = http.client.HTTPConnection("210.76.211.10")
+        
+    conn.request("GET", '/vplus/member/resource.do?isyulan=0&method=queryFlashXmlByResourceId&resourceId={resourceID}&randoms={randoms}'.format(resourceID = resourceID,
+                                                                                                                                            randoms = random()))
+    res = conn.getresponse()
+    data = res.read()
+
+    video_xml = data.decode("utf-8")
 
     root = ET.fromstring(video_xml.split('___!!!___')[0])
 

From fb04f289c2a60fae09299f39440002181cd7498d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 18 May 2017 03:15:55 +0800
Subject: [PATCH 0373/1225] [le]support sports.le

---
 src/you_get/extractors/le.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index bc034f3982..2cd5f16190 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -132,8 +132,14 @@ def letvcloud_download(url, output_dir='.', merge=True, info_only=False):
     letvcloud_download_by_vu(vu, uu, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
 def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
+    url = url_locations([url])[0]
     if re.match(r'http://yuntv.letv.com/', url):
         letvcloud_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
+    elif 'sports.le.com' in url:
+        html = get_content(url)
+        vid = match1(url, r'video/(\d+)\.html')
+        title = match1(html, r'<h2 class="title">([^<]+)</h2>')
+        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
     else:
         html = get_content(url)
         vid = match1(url, r'http://www.letv.com/ptv/vplay/(\d+).html') or \

From f1823eb16a4f693d9c06c2e8ede5bfd351a966bf Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 18 May 2017 03:50:07 +0800
Subject: [PATCH 0374/1225] [sina]try to find vid from url

---
 src/you_get/extractors/sina.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/sina.py b/src/you_get/extractors/sina.py
index fd45833570..121c5e13c1 100644
--- a/src/you_get/extractors/sina.py
+++ b/src/you_get/extractors/sina.py
@@ -77,6 +77,10 @@ def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
         vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
+        if vkey is None:
+            vid = match1(url, r'#(\d+)')
+            sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
+            return
         title = match1(video_page, r'title\s*:\s*"([^"]+)"')
         sina_download_by_vkey(vkey, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 

From d0a765572738870e9c7293cc641b8cd3998e586b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 May 2017 14:11:10 +0200
Subject: [PATCH 0375/1225] version 0.4.750

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index fd8d7bc010..23eaeb29da 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.715'
+__version__ = '0.4.750'

From 71a1825be5d935c3aa45ca81b12dbcabd3b95c52 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 24 May 2017 02:16:32 +0800
Subject: [PATCH 0376/1225] [acfun]fix youku cloud source

---
 src/you_get/common.py           | 23 +++++++++++++++
 src/you_get/extractors/acfun.py | 52 ++++++++++++++++++++++++++-------
 2 files changed, 65 insertions(+), 10 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8d4d2d7683..e98a80e901 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -138,6 +138,29 @@
 else:
     default_encoding = locale.getpreferredencoding().lower()
 
+def rc4(key, data):
+#all encryption algo should work on bytes
+    assert type(key)==type(data) and type(key) == type(b'')
+    state = list(range(256))
+    j = 0
+    for i in range(256):
+        j += state[i] + key[i % len(key)]
+        j &= 0xff
+        state[i], state[j] = state[j], state[i]
+
+    i = 0
+    j = 0
+    out_list = []
+    for char in data:
+        i += 1
+        i &= 0xff
+        j += state[i]
+        j &= 0xff
+        state[i], state[j] = state[j], state[i]
+        prn = state[(state[i] + state[j]) & 0xff]
+        out_list.append(char ^ prn)
+
+    return bytes(out_list)
 def maybe_print(*s):
     try: print(*s)
     except: pass
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 6bb0dca471..57ca616b62 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -10,11 +10,32 @@
 from .tudou import tudou_download_by_iid
 from .youku import youku_download_by_vid, youku_open_download_by_vid
 
-import json, re
+import json
+import re
+import base64
 
 def get_srt_json(id):
     url = 'http://danmu.aixifan.com/V2/%s' % id
-    return get_html(url)
+    return get_content(url)
+
+def youku_acfun_proxy(vid, sign):
+    url = 'http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid={}&ct=85&ev=2&sign={}'.format(vid, sign)
+    json_data = json.loads(get_content(url))['data']
+    enc_text = base64.b64decode(json_data)
+    dec_text = rc4(b'2da3ca9e', enc_text).decode('utf8')
+    youku_json = json.loads(dec_text)
+
+    yk_streams = {}
+    for stream in youku_json['stream']:
+        tp = stream['stream_type']
+        yk_streams[tp] = [], stream['total_size']
+        if stream.get('segs'):
+            for seg in stream['segs']:
+                yk_streams[tp][0].append(seg['url'])
+        else:
+            yk_streams[tp] = stream['m3u8'], stream['total_size']
+
+    return yk_streams
 
 def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
     """str, str, str, bool, bool ->None
@@ -26,7 +47,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     """
 
     #first call the main parasing API
-    info = json.loads(get_html('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
+    info = json.loads(get_content('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
 
     sourceType = info['sourceType']
 
@@ -47,12 +68,23 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
         #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
-        embsig =  info['encode']
-        a = 'http://api.aixifan.com/plays/%s' % vid
-        s = json.loads(get_content(a, headers={'deviceType': '2'}))
-        if s['data']['source'] == "zhuzhan-youku":
-            sourceId = s['data']['sourceId']
-            youku_open_download_by_vid(client_id='908a519d032263f8', vid=sourceId, title=title, output_dir=output_dir,merge=merge, info_only=info_only, embsig = embsig, **kwargs)
+#In Mar. 2017 there is a dedicated ``acfun_proxy'' in youku cloud player
+#old code removed
+        yk_streams = youku_acfun_proxy(info['sourceId'], info['encode'])
+        seq = ['mp4hd3', 'mp4hd2', 'mp4hd', 'flvhd']
+        for t in seq:
+            if yk_streams.get(t):
+                preferred = yk_streams[t]
+                break
+#total_size in the json could be incorrect(F.I. 0)
+        size = 0
+        for url in preferred[0]:
+            _, _, seg_size = url_info(url)
+            size += seg_size
+#fallback to flvhd is not quite possible
+        print_info(site_info, title, 'mp4', size)
+        if not info_only:
+            download_urls(preferred[0], title, 'mp4', size, output_dir=output_dir, merge=merge)
     else:
         raise NotImplementedError(sourceType)
 
@@ -71,7 +103,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
 
 def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     assert re.match(r'http://[^\.]+.acfun.[^\.]+/\D/\D\D(\d+)', url)
-    html = get_html(url)
+    html = get_content(url)
 
     title = r1(r'data-title="([^"]+)"', html)
     title = unescape_html(title)

From 5d38392cc3c0800025faef6576efc3195bb6d2be Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 25 May 2017 14:01:38 +0800
Subject: [PATCH 0377/1225] [youku tudou]update api

---
 src/you_get/extractors/tudou.py |   2 +-
 src/you_get/extractors/youku.py | 125 ++++++++++++--------------------
 2 files changed, 48 insertions(+), 79 deletions(-)

diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index ad37a191c2..b1568dfd13 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -61,7 +61,7 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         vcode = match1(html, r'viden\s*[:=]\s*\"([\w+/=]+)\"')
     if vcode:
         from .youku import youku_download_by_vid
-        return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return youku_download_by_vid(vcode, title=title, output_dir=output_dir, merge=merge, info_only=info_only, src='tudou', **kwargs)
 
     iid = r1(r'iid\s*[:=]\s*(\d+)', html)
     if not iid:
diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index c3ad67eb0e..845a0b42c0 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -8,6 +8,25 @@
 import ssl
 import time
 import traceback
+import json
+import urllib.request
+
+def fetch_cna():
+    if cookies:
+        for cookie in cookies:
+            if cookie.name == 'cna' and cookie.domain == '.youku.com':
+                log.i('Found cna in imported cookies. Use it')
+                return cookie.value
+    url = 'http://gm.mmstat.com/yt/ykcomment.play.commentInit?cna='
+    req = urllib.request.urlopen(url)
+    return req.info()['Set-Cookie'].split(';')[0].split('=')[1]
+
+def youku_ups(vid, ccode='0401'):
+    url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)
+    url += '&client_ip=192.168.1.1'
+    url += '&utid=' + fetch_cna()
+    url += '&client_ts=' + str(int(time.time()))
+    return json.loads(get_content(url))
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
@@ -154,66 +173,19 @@ def prepare(self, **kwargs):
             if self.vid is None:
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
-
-        #HACK!
-        if 'api_url' in kwargs:
-            api_url = kwargs['api_url']  #85
-            api12_url = kwargs['api12_url']  #86
-            self.ctype = kwargs['ctype']
-            self.title = kwargs['title']
-
+        if kwargs.get('src') and kwargs['src'] == 'tudou':
+            data = youku_ups(self.vid, '0402')['data']
         else:
-            api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=10' % self.vid
-            api12_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % self.vid
-
-        try:
-            meta = json.loads(get_content(
-                api_url,
-                headers={'Referer': 'http://static.youku.com/'}
-            ))
-            meta12 = json.loads(get_content(
-                api12_url,
-                headers={'Referer': 'http://static.youku.com/'}
-            ))
-            data = meta['data']
-            data12 = meta12['data']
-            assert 'stream' in data
-        except AssertionError:
-            if 'error' in data:
-                if data['error']['code'] == -202:
-                    # Password protected
-                    self.password_protected = True
-                    self.password = input(log.sprint('Password: ', log.YELLOW))
-                    api_url += '&pwd={}'.format(self.password)
-                    api12_url += '&pwd={}'.format(self.password)
-                    meta = json.loads(get_content(
-                        api_url,
-                        headers={'Referer': 'http://static.youku.com/'}
-                    ))
-                    meta12 = json.loads(get_content(
-                        api12_url,
-                        headers={'Referer': 'http://static.youku.com/'}
-                    ))
-                    data = meta['data']
-                    data12 = meta12['data']
-                else:
-                    log.wtf('[Failed] ' + data['error']['note'])
-            else:
-                log.wtf('[Failed] Video not found.')
-
-        if not self.title:  #86
-            self.title = data['video']['title']
-        self.ep = data12['security']['encrypt_string']
-        self.ip = data12['security']['ip']
-
-        if 'stream' not in data and self.password_protected:
-            log.wtf('[Failed] Wrong password.')
-
+            data = youku_ups(self.vid)['data']
+        if data.get('error'):
+            log.wtf(data['error']['note'])
+        self.title = data['video']['title']
         stream_types = dict([(i['id'], i) for i in self.stream_types])
         audio_lang = data['stream'][0]['audio_lang']
 
         for stream in data['stream']:
             stream_id = stream['stream_type']
+            is_preview = False
             if stream_id in stream_types and stream['audio_lang'] == audio_lang:
                 if 'alias-of' in stream_types[stream_id]:
                     stream_id = stream_types[stream_id]['alias-of']
@@ -227,40 +199,34 @@ def prepare(self, **kwargs):
                             'segs': stream['segs']
                         }]
                     }
+                    src = []
+                    for seg in stream['segs']:
+                        if seg.get('cdn_url'):
+                            src.append(seg['cdn_url'])
+                        else:
+                            is_preview = True
+                    self.streams[stream_id]['src'] = src
                 else:
                     self.streams[stream_id]['size'] += stream['size']
                     self.streams[stream_id]['pieces'].append({
                         'segs': stream['segs']
                     })
-
-        self.streams_fallback = {}
-        for stream in data12['stream']:
-            stream_id = stream['stream_type']
-            if stream_id in stream_types and stream['audio_lang'] == audio_lang:
-                if 'alias-of' in stream_types[stream_id]:
-                    stream_id = stream_types[stream_id]['alias-of']
-
-                if stream_id not in self.streams_fallback:
-                    self.streams_fallback[stream_id] = {
-                        'container': stream_types[stream_id]['container'],
-                        'video_profile': stream_types[stream_id]['video_profile'],
-                        'size': stream['size'],
-                        'pieces': [{
-                            'segs': stream['segs']
-                        }]
-                    }
-                else:
-                    self.streams_fallback[stream_id]['size'] += stream['size']
-                    self.streams_fallback[stream_id]['pieces'].append({
-                        'segs': stream['segs']
-                    })
+                    src = []
+                    for seg in stream['segs']:
+                        if seg.get('cdn_url'):
+                            src.append(seg['cdn_url'])
+                        else:
+                            is_preview = True
+                    self.streams[stream_id]['src'].extend(src)
+            if is_preview:
+                log.w('{} is a preview'.format(stream_id))
 
         # Audio languages
         if 'dvd' in data and 'audiolang' in data['dvd']:
             self.audiolang = data['dvd']['audiolang']
             for i in self.audiolang:
                 i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
-
+    '''
     def extract(self, **kwargs):
         if 'stream_id' in kwargs and kwargs['stream_id']:
             # Extract the stream
@@ -279,7 +245,6 @@ def extract(self, **kwargs):
             base64.b64decode(bytes(self.ep, 'ascii'))
         )
         sid, token = e_code.split('_')
-
         while True:
             try:
                 ksegs = []
@@ -327,6 +292,7 @@ def extract(self, **kwargs):
 
         if not kwargs['info_only']:
             self.streams[stream_id]['src'] = ksegs
+    '''
 
     def open_download_by_vid(self, client_id, vid, **kwargs):
         """self, str, str, **kwargs->None
@@ -394,3 +360,6 @@ def open_download_by_vid(self, client_id, vid, **kwargs):
 youku_download_by_vid = site.download_by_vid
 youku_open_download_by_vid = site.open_download_by_vid
 # Used by: acfun.py bilibili.py miomio.py tudou.py
+# acfun has its own proxy and won't use it
+# miomio is dead
+# tudou doesn't use ct85 so open_download_by_vid is uesless now.

From 0b00bf0f6213fedd470cfb1a1c343f7c6c20aae0 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 27 May 2017 00:45:42 +0800
Subject: [PATCH 0378/1225] [pptv]refactor

---
 src/you_get/extractors/pptv.py | 217 ++++++++++++++++++++++++++++++++-
 1 file changed, 211 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index b4ae78d1e3..9e55ac6c09 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -1,15 +1,218 @@
 #!/usr/bin/env python
 
-__all__ = ['pptv_download', 'pptv_download_by_id']
+#__all__ = ['pptv_download', 'pptv_download_by_id']
 
 from ..common import *
+from ..extractor import VideoExtractor
 
 import re
 import time
 import urllib
-from random import random
+import random
+import binascii
+from xml.dom.minidom import parseString
 
 
+def lshift(a, b):
+    return (a << b) & 0xffffffff
+def rshift(a, b):
+    if a >= 0:
+        return a >> b
+    return (0x100000000 + a) >> b
+
+def le32_pack(b_str):
+    result = 0
+    result |= b_str[0]
+    result |= (b_str[1] << 8)
+    result |= (b_str[2] << 16)
+    result |= (b_str[3] << 24)
+    return result
+
+def tea_core(data, key_seg):
+    delta = 2654435769
+
+    d0 = le32_pack(data[:4])
+    d1 = le32_pack(data[4:8])
+
+    sum_ = 0
+    for rnd in range(32):
+        sum_ = (sum_ + delta) & 0xffffffff
+        p1 = (lshift(d1, 4) + key_seg[0]) & 0xffffffff
+        p2 = (d1 + sum_) & 0xffffffff
+        p3 = (rshift(d1, 5) + key_seg[1]) & 0xffffffff
+
+        mid_p = p1 ^ p2 ^ p3
+        d0 = (d0 + mid_p) & 0xffffffff
+
+        p4 = (lshift(d0, 4) + key_seg[2]) & 0xffffffff
+        p5 = (d0 + sum_) & 0xffffffff
+        p6 = (rshift(d0, 5) + key_seg[3]) & 0xffffffff
+
+        mid_p = p4 ^ p5 ^ p6
+        d1 = (d1 + mid_p) & 0xffffffff
+
+    return bytes(unpack_le32(d0) + unpack_le32(d1))
+
+def ran_hex(size):
+    result = []
+    for i in range(size):
+        result.append(hex(int(15 * random.random()))[2:])
+    return ''.join(result)
+
+def zpad(b_str, size):
+    size_diff = size - len(b_str)
+    return b_str + bytes(size_diff)
+
+def gen_key(t):
+    key_seg = [1896220160,101056625, 100692230, 7407110]
+    t_s = hex(int(t))[2:].encode('utf8')
+    input_data = zpad(t_s, 16)
+    out = tea_core(input_data, key_seg)
+    return binascii.hexlify(out[:8]).decode('utf8') + ran_hex(16)
+
+def unpack_le32(i32):
+    result = []
+    result.append(i32 & 0xff)
+    i32 = rshift(i32, 8)
+    result.append(i32 & 0xff)
+    i32 = rshift(i32, 8)
+    result.append(i32 & 0xff)
+    i32 = rshift(i32, 8)
+    result.append(i32 & 0xff)
+    return result
+
+def get_elem(elem, tag):
+    return elem.getElementsByTagName(tag)
+
+def get_attr(elem, attr):
+    return elem.getAttribute(attr)
+
+def get_text(elem):
+    return elem.firstChild.nodeValue
+
+def shift_time(time_str):
+    ts = time_str[:-4]
+    return time.mktime(time.strptime(ts)) - 60
+
+def parse_pptv_xml(dom):
+    channel = get_elem(dom, 'channel')[0]
+    title = get_attr(channel, 'nm')
+    file_list = get_elem(channel, 'file')[0]
+    item_list = get_elem(file_list, 'item')
+    streams_cnt = len(item_list)
+    item_mlist = []
+    for item in item_list:
+        rid = get_attr(item, 'rid')
+        file_type = get_attr(item, 'ft')
+        size = get_attr(item, 'filesize')
+        width = get_attr(item, 'width')
+        height = get_attr(item, 'height')
+        bitrate = get_attr(item, 'bitrate')
+        res = '{}x{}@{}kbps'.format(width, height, bitrate)
+        item_meta = (file_type, rid, size, res)
+        item_mlist.append(item_meta)
+
+    dt_list = get_elem(dom, 'dt')
+    dragdata_list = get_elem(dom, 'dragdata')
+
+    stream_mlist = []
+    for dt in dt_list:
+        file_type = get_attr(dt, 'ft')
+        serv_time = get_text(get_elem(dt, 'st')[0])
+        expr_time = get_text(get_elem(dt, 'key')[0])
+        serv_addr = get_text(get_elem(dt, 'sh')[0])
+        stream_meta = (file_type, serv_addr, expr_time, serv_time)
+        stream_mlist.append(stream_meta)
+
+    segs_mlist = []
+    for dd in dragdata_list:
+        file_type = get_attr(dd, 'ft')
+        seg_list = get_elem(dd, 'sgm')
+        segs = []
+        segs_size = []
+        for seg in seg_list:
+            rid = get_attr(seg, 'rid')
+            size = get_attr(seg, 'fs')
+            segs.append(rid)
+            segs_size.append(size)
+        segs_meta = (file_type, segs, segs_size)
+        segs_mlist.append(segs_meta)
+    return title, item_mlist, stream_mlist, segs_mlist
+
+#mergs 3 meta_data
+def merge_meta(item_mlist, stream_mlist, segs_mlist):
+    streams = {}
+    for i in range(len(segs_mlist)):
+        streams[str(i)] = {}
+
+    for item in item_mlist:
+        stream = streams[item[0]]
+        stream['rid'] = item[1]
+        stream['size'] = item[2]
+        stream['res'] = item[3]
+
+    for s in stream_mlist:
+        stream = streams[s[0]]
+        stream['serv_addr'] = s[1]
+        stream['expr_time'] = s[2]
+        stream['serv_time'] = s[3]
+
+    for seg in segs_mlist:
+        stream = streams[seg[0]]
+        stream['segs'] = seg[1]
+        stream['segs_size'] = seg[2]
+
+    return streams
+
+
+def make_url(stream):
+    host = stream['serv_addr']
+    rid = stream['rid']
+    key = gen_key(shift_time(stream['serv_time']))
+    key_expr = stream['expr_time']
+
+    src = []
+    for i, seg in enumerate(stream['segs']):
+        url = 'http://{}/{}/{}?key={}&k={}'.format(host, i, rid, key, key_expr)
+        url += '&fpp.ver=1.3.0.4&type=web.fpp'
+        src.append(url)
+    return src
+
+class PPTV(VideoExtractor):
+    name = 'PPTV'
+    stream_types = [
+            {'itag': '4'},
+            {'itag': '3'},
+            {'itag': '2'},
+            {'itag': '1'},
+            {'itag': '0'},
+    ]
+
+    def prepare(self, **kwargs):
+        if self.url and not self.vid:
+            if not re.match(r'http://v.pptv.com/show/(\w+)\.html', self.url):
+                raise('Unknown url pattern')
+            page_content = get_content(self.url)
+            self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)')
+
+        if not self.vid:
+            raise('Cannot find id')
+        api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid)
+        api_url += '?type=web.fpp&version=4'
+        dom = parseString(get_content(api_url))
+        self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom)
+        xml_streams = merge_meta(m_items, m_streams, m_segs)
+        for stream_id in xml_streams:
+            stream_data = xml_streams[stream_id]
+            src = make_url(stream_data)
+            self.streams[stream_id] = {
+                    'container': 'mp4',
+                    'video_profile': stream_data['res'],
+                    'size': int(stream_data['size']),
+                    'src': src
+            }
+
+'''
 def constructKey(arg):
 
     def str2hex(s):
@@ -143,12 +346,14 @@ def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_o
             pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def pptv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    assert re.match(r'http://v.pptv.com/show/(\w+)\.html$', url)
+    assert re.match(r'http://v.pptv.com/show/(\w+)\.html', url)
     html = get_html(url)
     id = r1(r'webcfg\s*=\s*{"id":\s*(\d+)', html)
     assert id
     pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
-
-site_info = "PPTV.com"
-download = pptv_download
+'''
+site = PPTV()
+#site_info = "PPTV.com"
+#download = pptv_download
+download = site.download_by_url
 download_playlist = playlist_not_supported('pptv')

From da1339b0b035be7a176605295215d9ad2103d715 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 27 May 2017 20:14:52 +0800
Subject: [PATCH 0379/1225] [acfun]update interface

---
 src/you_get/extractors/acfun.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 57ca616b62..95c66d80ac 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -19,10 +19,10 @@ def get_srt_json(id):
     return get_content(url)
 
 def youku_acfun_proxy(vid, sign):
-    url = 'http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid={}&ct=85&ev=2&sign={}'.format(vid, sign)
+    url = 'http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid={}&ct=85&ev=3&sign={}'.format(vid, sign)
     json_data = json.loads(get_content(url))['data']
     enc_text = base64.b64decode(json_data)
-    dec_text = rc4(b'2da3ca9e', enc_text).decode('utf8')
+    dec_text = rc4(b'8bdc7e1a', enc_text).decode('utf8')
     youku_json = json.loads(dec_text)
 
     yk_streams = {}

From 91b196baef0414858a2614d64b32269e1a7f4c78 Mon Sep 17 00:00:00 2001
From: YenvY <dusk.embrace@gmail.com>
Date: Wed, 14 Jun 2017 09:06:51 +0800
Subject: [PATCH 0380/1225] rebuild flickr extractor

---
 src/you_get/extractors/flickr.py | 239 +++++++++++++++++++++++++++----
 1 file changed, 214 insertions(+), 25 deletions(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index b0f102f6ee..8d55a8960d 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -1,39 +1,228 @@
 #!/usr/bin/env python
 
-__all__ = ['flickr_download']
+__all__ = ['flickr_download_main']
 
 from ..common import *
 
-def flickr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    page = get_html(url)
-    title = match1(page, r'<meta property="og:title" content="([^"]*)"')
-    photo_id = match1(page, r'"id":"([0-9]+)"')
+import json
 
-    try: # extract video
-        html = get_html('https://secure.flickr.com/apps/video/video_mtl_xml.gne?photo_id=%s' % photo_id)
-        node_id = match1(html, r'<Item id="id">(.+)</Item>')
-        secret = match1(html, r'<Item id="photo_secret">(.+)</Item>')
+pattern_url_photoset = r'https?://www\.flickr\.com/photos/.+/(?:(?:sets)|(?:albums))?/([^/]+)'
+pattern_url_photostream = r'https?://www\.flickr\.com/photos/([^/]+)(?:/|(?:/page))?$'
+pattern_url_single_photo = r'https?://www\.flickr\.com/photos/[^/]+/(\d+)'
+pattern_url_gallery = r'https?://www\.flickr\.com/photos/[^/]+/galleries/(\d+)'
+pattern_url_group = r'https?://www\.flickr\.com/groups/([^/]+)'
+pattern_url_favorite = r'https?://www\.flickr\.com/photos/([^/]+)/favorites'
 
-        html = get_html('https://secure.flickr.com/video_playlist.gne?node_id=%s&secret=%s' % (node_id, secret))
-        app = match1(html, r'APP="([^"]+)"')
-        fullpath = unescape_html(match1(html, r'FULLPATH="([^"]+)"'))
-        url = app + fullpath
+pattern_inline_title = r'<title>([^<]*)</title>'
+pattern_inline_api_key = r'api\.site_key\s*=\s*"([^"]+)"'
+pattern_inline_img_url = r'"url":"([^"]+)","key":"[^"]+"}}'
+pattern_inline_NSID = r'"nsid"\s*:\s*"([^"]+)"'
+pattern_inline_video_mark = r'("mediaType":"video")'
 
-        mime, ext, size = url_info(url)
+# (api_key, method, ext, page)
+tmpl_api_call = (
+    'https://api.flickr.com/services/rest?'
+    '&format=json&nojsoncallback=1'
+    # UNCOMMENT FOR TESTING
+    #'&per_page=5'
+    '&per_page=500'
+    # this parameter CANNOT take control of 'flickr.galleries.getPhotos'
+    # though the doc said it should.
+    # it's always considered to be 500
+    '&api_key=%s'
+    '&method=flickr.%s'
+    '&extras=url_sq,url_q,url_t,url_s,url_n,url_m,url_z,url_c,url_l,url_h,url_k,url_o,media'
+    '%s&page=%d'
+)
 
-        print_info(site_info, title, mime, size)
-        if not info_only:
-            download_urls([url], title, ext, size, output_dir, merge=merge, faker=True)
+tmpl_api_call_video_info = (
+    'https://api.flickr.com/services/rest?'
+    '&format=json&nojsoncallback=1'
+    '&method=flickr.video.getStreamInfo'
+    '&api_key=%s'
+    '&photo_id=%s'
+    '&secret=%s'
+)
+
+tmpl_api_call_photo_info = (
+    'https://api.flickr.com/services/rest?'
+    '&format=json&nojsoncallback=1'
+    '&method=flickr.photos.getInfo'
+    '&api_key=%s'
+    '&photo_id=%s'
+)
+
+# looks that flickr won't return urls for all sizes
+# we required in 'extras field without a acceptable header
+dummy_header = {
+    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:53.0) Gecko/20100101 Firefox/53.0'
+}
+def get_content_headered(url):
+    return get_content(url, dummy_header)
+
+def get_photoset_id(url, page):
+    return match1(url, pattern_url_photoset)
+
+def get_photo_id(url, page):
+    return match1(url, pattern_url_single_photo)
+
+def get_gallery_id(url, page):
+    return match1(url, pattern_url_gallery)
+
+def get_api_key(page):
+    match = match1(page, pattern_inline_api_key)
+    # this happens only when the url points to a gallery page
+    # that contains no inline api_key(and never makes xhr api calls)
+    # in fact this might be a better approch for getting a temporary api key
+    # since there's no place for a user to add custom infomation that may
+    # misguide the regex in the homepage
+    if not match:
+        return match1(get_html('https://flickr.com'), r'"site_key"\s*:\s*"([^"]+)"')
+    return match
+
+def get_NSID(url, page):
+    return match1(page, pattern_inline_NSID)
 
-    except: # extract images
-        image = match1(page, r'<meta property="og:image" content="([^"]*)')
-        ext = 'jpg'
-        _, _, size = url_info(image)
+# [
+# (
+#   regex_match_url,
+#   remote_api_method,
+#   additional_query_parameter_for_method,
+#   parser_for_additional_parameter,
+#   field_where_photourls_are_saved
+# )
+# ]
+url_patterns = [
+    # www.flickr.com/photos/{username|NSID}/sets|albums/{album-id}
+    (
+        pattern_url_photoset,
+        'photosets.getPhotos',
+        'photoset_id',
+        get_photoset_id,
+        'photoset'
+    ),
+    # www.flickr.com/photos/{username|NSID}/{pageN}?
+    (
+        pattern_url_photostream,
+        # according to flickr api documentation, this method needs to be
+        # authenticated in order to filter photo visible to the calling user
+        # but it seems works fine anonymously as well
+        'people.getPhotos',
+        'user_id',
+        get_NSID,
+        'photos'
+    ),
+    # www.flickr.com/photos/{username|NSID}/galleries/{gallery-id}
+    (
+        pattern_url_gallery,
+        'galleries.getPhotos',
+        'gallery_id',
+        get_gallery_id,
+        'photos'
+    ),
+    # www.flickr.com/groups/{groupname|groupNSID}/
+    (
+        pattern_url_group,
+        'groups.pools.getPhotos',
+        'group_id',
+        get_NSID,
+        'photos'
+    ),
+    # www.flickr.com/photos/{username|NSID}/favorites/*
+    (
+        pattern_url_favorite,
+        'favorites.getList',
+        'user_id',
+        get_NSID,
+        'photos'
+    )
+]
 
-        print_info(site_info, title, ext, size)
+def flickr_download_main(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    urls = None
+    size = 'o' # works for collections only
+    title = None
+    if 'stream_id' in kwargs:
+        size = kwargs['stream_id']
+    if match1(url, pattern_url_single_photo):
+        url, title = get_single_photo_url(url)
+        urls = [url]
+    else:
+        urls, title = fetch_photo_url_list(url, size)
+    index = 0
+    for url in urls:
+        mime, ext, size = url_info(url)
+        print_info('Flickr.com', title, mime, size)
         if not info_only:
-            download_urls([image], title, ext, size, output_dir, merge=merge)
+            suffix = '[%d]' % index
+            download_urls([url], title + suffix, ext, False, output_dir, None, False, False)
+            index = index + 1
+
+def fetch_photo_url_list(url, size):
+    for pattern in url_patterns:
+        # FIXME: fix multiple matching since the match group is dropped
+        if match1(url, pattern[0]):
+            return fetch_photo_url_list_impl(url, *pattern[1:], size)
+    raise NotImplementedError('Flickr extractor is not supported for %s.' % url)
+
+def fetch_photo_url_list_impl(url, method, id_field, id_parse_func, collection_name, size):
+    page = get_html(url)
+    api_key = get_api_key(page)
+    ext_field = ''
+    if id_parse_func:
+        ext_field = '&%s=%s' % (id_field, id_parse_func(url, page))
+    page_number = 1
+    urls = []
+    while True:
+        call_url = tmpl_api_call % (api_key, method, ext_field, page_number)
+        photoset = json.loads(get_content_headered(call_url))[collection_name]
+        pagen = photoset['page']
+        pages = photoset['pages']
+        for info in photoset['photo']:
+            url = get_url_of_largest(info, api_key, size)
+            urls.append(url)
+        page_number = page_number + 1
+        # the typeof 'page' and 'pages' may change in different methods
+        if str(pagen) == str(pages):
+            break
+    return urls, match1(page, pattern_inline_title)
+
+# image size suffixes used in inline json 'key' field
+# listed in descending order
+size_suffixes = ['o', 'k', 'h', 'l', 'c', 'z', 'm', 'n', 's', 't', 'q', 'sq']
+
+def get_orig_video_source(api_key, pid, secret):
+    parsed = json.loads(get_content_headered(tmpl_api_call_video_info % (api_key, pid, secret)))
+    for stream in parsed['streams']['stream']:
+        if stream['type'] == 'orig':
+            return stream['_content'].replace('\\', '')
+    return None
+
+def get_url_of_largest(info, api_key, size):
+    if info['media'] == 'photo':
+        sizes = size_suffixes
+        if size in sizes:
+            sizes = sizes[sizes.index(size):]
+        for suffix in sizes:
+            if 'url_' + suffix in info:
+                return info['url_' + suffix].replace('\\', '')
+        return None
+    else:
+        return get_orig_video_source(api_key, info['id'], info['secret'])
+
+def get_single_photo_url(url):
+    page = get_html(url)
+    pid = get_photo_id(url, page)
+    title = match1(page, pattern_inline_title)
+    if match1(page, pattern_inline_video_mark):
+        api_key = get_api_key(page)
+        reply = get_content(tmpl_api_call_photo_info % (api_key, get_photo_id(url, page)))
+        secret = json.loads(reply)['photo']['secret']
+        return get_orig_video_source(api_key, pid, secret), title
+    #last match always has the best resolution
+    match = match1(page, pattern_inline_img_url)
+    return 'https:' + match.replace('\\', ''), title
 
 site_info = "Flickr.com"
-download = flickr_download
-download_playlist = playlist_not_supported('flickr')
+download = flickr_download_main
+download_playlist = playlist_not_supported('flickr');

From 73f936974c6af77e486bf2935a548e43d10117e0 Mon Sep 17 00:00:00 2001
From: YenvY <dusk.embrace@gmail.com>
Date: Wed, 14 Jun 2017 11:26:24 +0800
Subject: [PATCH 0381/1225] fix for lower version python

---
 src/you_get/extractors/flickr.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index 8d55a8960d..f91d088415 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -162,10 +162,10 @@ def fetch_photo_url_list(url, size):
     for pattern in url_patterns:
         # FIXME: fix multiple matching since the match group is dropped
         if match1(url, pattern[0]):
-            return fetch_photo_url_list_impl(url, *pattern[1:], size)
+            return fetch_photo_url_list_impl(url, size, *pattern[1:])
     raise NotImplementedError('Flickr extractor is not supported for %s.' % url)
 
-def fetch_photo_url_list_impl(url, method, id_field, id_parse_func, collection_name, size):
+def fetch_photo_url_list_impl(url, size, method, id_field, id_parse_func, collection_name):
     page = get_html(url)
     api_key = get_api_key(page)
     ext_field = ''

From cbe4fe09dc516a394c55965e7e9ae487cff0b113 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 17 Jun 2017 01:46:31 +0800
Subject: [PATCH 0382/1225] rewrite qq_download_by_vid

---
 src/you_get/extractors/qq.py | 101 ++++++++++++-----------------------
 1 file changed, 34 insertions(+), 67 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 4a9d5f898d..e7d5f146ca 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -3,75 +3,42 @@
 __all__ = ['qq_download']
 
 from ..common import *
+from ..util.log import *
 from .qie import download as qieDownload
 from urllib.parse import urlparse,parse_qs
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3%2E2%2E19%2E333&platform=11&defnpayver=1&vid=' + vid
-    info = get_html(info_api)
+    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform=11&defnpayver=1&vid={}'.format(vid)
+    info = get_content(info_api)
     video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
-    parts_vid = video_json['vl']['vi'][0]['vid']
-    parts_ti = video_json['vl']['vi'][0]['ti']
-    parts_prefix = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
-    parts_formats = video_json['fl']['fi']
-    if parts_prefix.endswith('/'):
-        parts_prefix = parts_prefix[:-1]
-    # find best quality
-    # only looking for fhd(1080p) and shd(720p) here.
-    # 480p usually come with a single file, will be downloaded as fallback.
-    best_quality = ''
-    for part_format in parts_formats:
-        if part_format['name'] == 'fhd':
-            best_quality = 'fhd'
-            break
-
-        if part_format['name'] == 'shd':
-            best_quality = 'shd'
-
-    for part_format in parts_formats:
-        if (not best_quality == '') and (not part_format['name'] == best_quality):
-            continue
-        part_format_id = part_format['id']
-        part_format_sl = part_format['sl']
-        if part_format_sl == 0:
-            part_urls= []
-            total_size = 0
-            try:
-                # For fhd(1080p), every part is about 100M and 6 minutes
-                # try 100 parts here limited download longest single video of 10 hours.
-                for part in range(1,100):
-                    filename = vid + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
-                    key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format=%s&vid=%s&filename=%s" % (part_format_id, parts_vid, filename)
-                    #print(filename)
-                    #print(key_api)
-                    part_info = get_html(key_api)
-                    key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
-                    #print(key_json)
-                    vkey = key_json['key']
-                    url = '%s/%s?vkey=%s' % (parts_prefix, filename, vkey)
-                    part_urls.append(url)
-                    _, ext, size = url_info(url, faker=True)
-                    total_size += size
-            except:
-                pass
-            print_info(site_info, parts_ti, ext, total_size)
-            if not info_only:
-                download_urls(part_urls, parts_ti, ext, total_size, output_dir=output_dir, merge=merge)
-        else:
-            fvkey = video_json['vl']['vi'][0]['fvkey']
-            mp4 = video_json['vl']['vi'][0]['cl'].get('ci', None)
-            if mp4:
-                old_id = mp4[0]['keyid'].split('.')[1]
-                new_id = 'p' + str(int(old_id) % 10000)
-                mp4 = mp4[0]['keyid'].replace(old_id, new_id) + '.mp4'
-            else:
-                mp4 = video_json['vl']['vi'][0]['fn']
-            url = '%s/%s?vkey=%s' % ( parts_prefix, mp4, fvkey )
-            _, ext, size = url_info(url, faker=True)
-
-            print_info(site_info, title, ext, size)
-            if not info_only:
-                download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
+
+    if video_json['exem'] != 0:
+        log.wtf(video_json['msg'])
+    fn_pre = video_json['vl']['vi'][0]['lnk']
+    title = video_json['vl']['vi'][0]['ti']
+    host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
+    streams = video_json['fl']['fi']
+    seg_cnt = len(video_json['vl']['vi'][0]['cl']['ci'])
+
+    best_quality = streams[-1]['name']
+    part_format_id = streams[-1]['id']
+
+    part_urls= []
+    total_size = 0
+    for part in range(1, seg_cnt+1):
+        filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+        key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
+        part_info = get_content(key_api)
+        key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
+        vkey = key_json['key']
+        url = '{}{}?vkey={}'.format(host, filename, vkey)
+        part_urls.append(url)
+        _, ext, size = url_info(url)
+        total_size += size
+
+    print_info(site_info, title, ext, total_size)
+    if not info_only:
+        download_urls(part_urls, title, ext, total_size, output_dir=output_dir, merge=merge)
 
 def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=False):
     BASE_URL = 'http://cgi.kg.qq.com/fcgi-bin/kg_ugc_getdetail'
@@ -124,7 +91,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         return
 
     if 'mp.weixin.qq.com/s?' in url:
-        content = get_html(url)
+        content = get_content(url)
         vids = matchall(content, [r'\bvid=(\w+)'])
         for vid in vids:
             qq_download_by_vid(vid, vid, output_dir, merge, info_only)
@@ -143,7 +110,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             url = new_url
 
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
-        content = get_html(url)
+        content = get_content(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
@@ -152,7 +119,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         # for embedded URLs; don't know what the title is
         title = vid
     else:
-        content = get_html(url)
+        content = get_content(url)
         vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
         vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)

From 8e4080944658a1113f6114aaf4b2343ac86fc75d Mon Sep 17 00:00:00 2001
From: yunair <airzhaoyn@gmail.com>
Date: Mon, 19 Jun 2017 14:16:50 +0800
Subject: [PATCH 0383/1225] fix miaopai mobile page download url does not
 contains .htm

---
 src/you_get/extractors/yixia.py |  4 +++-
 tests/test.py                   | 13 +++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index 7d5ba29089..13b56a3f15 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -57,7 +57,9 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
             scid = match1(url, r'http://www.miaopai.com/show/(.+)\.htm')
         elif re.match(r'http://m.miaopai.com/show/channel/.+', url):  #Mobile
             scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)\.htm')
-    
+            if scid == None :
+                scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)')
+
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu
         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid
         site_info = "Yixia Xiaokaxiu"
diff --git a/tests/test.py b/tests/test.py
index ba15e4478a..84fb9cae66 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -2,12 +2,10 @@
 
 import unittest
 
-from you_get import *
 from you_get.extractors import *
-from you_get.common import *
 
-class YouGetTests(unittest.TestCase):
 
+class YouGetTests(unittest.TestCase):
     def test_imgur(self):
         imgur.download("http://imgur.com/WVLk5nD", info_only=True)
         imgur.download("http://imgur.com/gallery/WVLk5nD", info_only=True)
@@ -18,4 +16,11 @@ def test_magisto(self):
     def test_youtube(self):
         youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
         youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
-        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare", info_only=True)
+        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare",
+                         info_only=True)
+
+    def test_yixia(self):
+        yixia_download("http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8", info_only=True)
+
+if __name__ == '__main__':
+    unittest.main()

From ca7d03357ef68cde258ef08a45d8dd30c1935c7b Mon Sep 17 00:00:00 2001
From: w48325832 <w48325832@gmail.com>
Date: Sun, 25 Jun 2017 14:20:28 +0800
Subject: [PATCH 0384/1225] add iwara

---
 src/you_get/common.py           |  1 +
 src/you_get/extractors/iwara.py | 40 +++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 src/you_get/extractors/iwara.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8d4d2d7683..a7ae299e96 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -92,6 +92,7 @@
     'miaopai'          : 'yixia',
     'yizhibo'          : 'yizhibo',
     'youku'            : 'youku',
+    'iwara'            : 'iwara',
     'youtu'            : 'youtube',
     'youtube'          : 'youtube',
     'zhanqi'           : 'zhanqi',
diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
new file mode 100644
index 0000000000..1f737c12fa
--- /dev/null
+++ b/src/you_get/extractors/iwara.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+__all__ = ['iwara_download']
+from ..common import *
+from ..common import print_more_compatible as print
+from ..extractor import VideoExtractor
+from ..util import log
+from .. import json_output
+headers = {
+    'DNT': '1',
+    'Accept-Encoding': 'gzip, deflate, sdch, br',
+    'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
+    'Upgrade-Insecure-Requests': '1',
+    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+    'Cache-Control': 'max-age=0',
+    'Referer': 'http://www.dilidili.com/',
+    'Connection': 'keep-alive',
+    'Save-Data': 'on',
+}
+
+
+
+def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    global headers
+    video_hash=match1(url, r'http://ecchi.iwara.tv/videos/(\w+)')
+    html = get_html(url)
+    title = r1(r'<title>(.*)</title>', html)
+    api_url='http://ecchi.iwara.tv/api/video/'+video_hash
+    content=get_html(api_url)
+    mime='video/mp4'
+    size=0;
+    print_info(content,title,mime,size)
+
+    print_info(site_info, title, mime, size)
+    if not info_only:
+        download_urls([source], title, ext, size, output_dir, merge=merge)
+
+site_info = "iwara"
+download = iwara_download
+download_playlist = playlist_not_supported('archive')
\ No newline at end of file

From 12a842dd887e3bfb0ae27c1df771d3003ad3703f Mon Sep 17 00:00:00 2001
From: w48325832 <w48325832@gmail.com>
Date: Mon, 26 Jun 2017 23:59:47 +0800
Subject: [PATCH 0385/1225] add iwara support

---
 src/you_get/extractors/iwara.py | 26 ++++++++++----------------
 1 file changed, 10 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 1f737c12fa..2a3649b381 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -1,10 +1,6 @@
 #!/usr/bin/env python
 __all__ = ['iwara_download']
 from ..common import *
-from ..common import print_more_compatible as print
-from ..extractor import VideoExtractor
-from ..util import log
-from .. import json_output
 headers = {
     'DNT': '1',
     'Accept-Encoding': 'gzip, deflate, sdch, br',
@@ -13,28 +9,26 @@
     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
     'Cache-Control': 'max-age=0',
-    'Referer': 'http://www.dilidili.com/',
+
     'Connection': 'keep-alive',
     'Save-Data': 'on',
+    'Cookie':'has_js=1;show_adult=1',
 }
 
-
-
 def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     global headers
     video_hash=match1(url, r'http://ecchi.iwara.tv/videos/(\w+)')
-    html = get_html(url)
+    html = get_content(url,headers=headers)
     title = r1(r'<title>(.*)</title>', html)
     api_url='http://ecchi.iwara.tv/api/video/'+video_hash
-    content=get_html(api_url)
-    mime='video/mp4'
-    size=0;
-    print_info(content,title,mime,size)
+    content=get_content(api_url,headers=headers)
+    data=json.loads(content)
+    type,ext,size=url_info(data[0]['uri'], headers=headers)
+    down_urls=data[0]['uri']
+    print_info(down_urls,title+data[0]['resolution'],type,size)
 
-    print_info(site_info, title, mime, size)
-    if not info_only:
-        download_urls([source], title, ext, size, output_dir, merge=merge)
+    download_urls([down_urls], title, ext, size, output_dir, merge = merge,headers=headers)
 
 site_info = "iwara"
 download = iwara_download
-download_playlist = playlist_not_supported('archive')
\ No newline at end of file
+download_playlist = playlist_not_supported('iwara')
\ No newline at end of file

From 82b4496dc9e28be2c98fc57354be9a558346fd8f Mon Sep 17 00:00:00 2001
From: w48325832 <w48325832@gmail.com>
Date: Tue, 27 Jun 2017 00:13:35 +0800
Subject: [PATCH 0386/1225] fix www.iwara.tv can't download

---
 src/you_get/extractors/iwara.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 2a3649b381..21b44608d8 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -17,10 +17,11 @@
 
 def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     global headers
-    video_hash=match1(url, r'http://ecchi.iwara.tv/videos/(\w+)')
+    video_hash=match1(url, r'http://\w+.iwara.tv/videos/(\w+)')
+    video_url=match1(url, r'(http://\w+.iwara.tv)/videos/\w+')
     html = get_content(url,headers=headers)
     title = r1(r'<title>(.*)</title>', html)
-    api_url='http://ecchi.iwara.tv/api/video/'+video_hash
+    api_url=video_url+'/api/video/'+video_hash
     content=get_content(api_url,headers=headers)
     data=json.loads(content)
     type,ext,size=url_info(data[0]['uri'], headers=headers)

From ec79b7eccd0c7ebbe2f9d806075195f6005f6e23 Mon Sep 17 00:00:00 2001
From: YenvY <dusk.embrace@gmail.com>
Date: Mon, 3 Jul 2017 07:49:23 +0800
Subject: [PATCH 0387/1225] fix 'reference before declaration' error in youtube
 extractor

---
 src/you_get/extractors/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index d08c848ac6..da80a650a7 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -144,7 +144,7 @@ def prepare(self, **kwargs):
 
         if 'status' not in video_info:
             log.wtf('[Failed] Unknown status.')
-
+        ytplayer_config = None
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
                 self.title = parse.unquote_plus(video_info['title'][0])
@@ -199,7 +199,7 @@ def prepare(self, **kwargs):
             log.wtf('[Failed] Invalid status.')
 
         # YouTube Live
-        if ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1':
+        if ytplayer_config and (ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1'):
             hlsvp = ytplayer_config['args']['hlsvp']
 
             if 'info_only' in kwargs and kwargs['info_only']:

From f4e8544bd365ab23dd96c8ea88f7d25d7129e69e Mon Sep 17 00:00:00 2001
From: YenvY <dusk.embrace@gmail.com>
Date: Mon, 3 Jul 2017 08:02:36 +0800
Subject: [PATCH 0388/1225] fix an idiot mistake

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index da80a650a7..c79879563b 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -142,9 +142,9 @@ def prepare(self, **kwargs):
 
         video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}'.format(self.vid)))
 
+        ytplayer_config = None
         if 'status' not in video_info:
             log.wtf('[Failed] Unknown status.')
-        ytplayer_config = None
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
                 self.title = parse.unquote_plus(video_info['title'][0])

From 5d19825baaa443b3aa213eb2afc94e21de9dcc61 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 3 Jul 2017 13:37:58 +0800
Subject: [PATCH 0389/1225] [bilibili]rewrite

---
 src/you_get/extractor.py           |  25 +-
 src/you_get/extractors/bilibili.py | 468 +++++++++++++++++------------
 2 files changed, 301 insertions(+), 192 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index af7cc824f2..1a68dbafd3 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy
+from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy, get_content, dry_run
 from .common import print_more_compatible as print
 from .util import log
 from . import json_output
@@ -28,6 +28,10 @@ def __init__(self, *args):
         self.password_protected = False
         self.dash_streams = {}
         self.caption_tracks = {}
+        self.out = False
+        self.ua = None
+        self.referer = None
+        self.danmuku = None
 
         if args:
             self.url = args[0]
@@ -39,6 +43,8 @@ def download_by_url(self, url, **kwargs):
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             set_proxy(parse_host(kwargs['extractor_proxy']))
         self.prepare(**kwargs)
+        if self.out:
+            return
         if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
             unset_proxy()
 
@@ -99,7 +105,8 @@ def p_stream(self, stream_id):
             print("      quality:       %s" % stream['quality'])
 
         if 'size' in stream and stream['container'].lower() != 'm3u8':
-            print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+            if stream['size'] != float('inf')  and stream['size'] != 0:
+                print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
 
         if 'itag' in stream:
             print("    # download-with: %s" % log.sprint("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
@@ -202,12 +209,17 @@ def download(self, **kwargs):
             if not urls:
                 log.wtf('[Failed] Cannot extract video source.')
             # For legacy main()
-            download_urls(urls, self.title, ext, total_size,
+            headers = {}
+            if self.ua is not None:
+                headers['User-Agent'] = self.ua
+            if self.referer is not None:
+                headers['Referer'] = self.referer
+            download_urls(urls, self.title, ext, total_size, headers=headers,
                           output_dir=kwargs['output_dir'],
                           merge=kwargs['merge'],
                           av=stream_id in self.dash_streams)
             if 'caption' not in kwargs or not kwargs['caption']:
-                print('Skipping captions.')
+                print('Skipping captions or danmuku.')
                 return
             for lang in self.caption_tracks:
                 filename = '%s.%s.srt' % (get_filename(self.title), lang)
@@ -217,6 +229,11 @@ def download(self, **kwargs):
                           'w', encoding='utf-8') as x:
                     x.write(srt)
                 print('Done.')
+            if self.danmuku is not None and not dry_run:
+                filename = '{}.cmt.xml'.format(get_filename(self.title))
+                print('Downloading {} ...\n'.format(filename))
+                with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
+                    fp.write(self.danmuku)
 
             # For main_dev()
             #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index ccb395cb56..8b18eeabc4 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -2,210 +2,302 @@
 
 __all__ = ['bilibili_download']
 
+import hashlib
+import re
+import time
+import json
+import http.cookiejar
+import urllib.request
+import urllib.parse
+from xml.dom.minidom import parseString
+
 from ..common import *
+from ..util.log import *
+from ..extractor import *
 
+from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
 from .youku import youku_download_by_vid
 
-import hashlib
-import re
-
-appkey = 'f3bb208b3d081dc8'
-SECRETKEY_MINILOADER = '1c15888dc316e05a15fdd0a02ed6584f'
-
-def get_srt_xml(id):
-    url = 'http://comment.bilibili.com/%s.xml' % id
-    return get_html(url)
-
-
-def parse_srt_p(p):
-    fields = p.split(',')
-    assert len(fields) == 8, fields
-    time, mode, font_size, font_color, pub_time, pool, user_id, history = fields
-    time = float(time)
-
-    mode = int(mode)
-    assert 1 <= mode <= 8
-    # mode 1~3: scrolling
-    # mode 4: bottom
-    # mode 5: top
-    # mode 6: reverse?
-    # mode 7: position
-    # mode 8: advanced
+class Bilibili(VideoExtractor):
+    name = 'Bilibili'
+    live_api = 'http://live.bilibili.com/api/playurl?cid={}&otype=json'
+    api_url = 'http://interface.bilibili.com/playurl?'
+    bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
+    
+    SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
+    SEC2 = '9b288147e5474dd2aa67085f716c560d'
+    stream_types = [
+            {'id': 'hdflv'},
+            {'id': 'flv'},
+            {'id': 'hdmp4'},
+            {'id': 'mp4'},
+            {'id': 'live'}
+    ]
+    fmt2qlt = dict(hdflv=4, flv=3, hdmp4=2, mp4=1)
+
+    @staticmethod
+    def bilibili_stream_type(urls):
+        url = urls[0]
+        if 'hd.flv?' in url:
+            return 'hdflv', 'flv'
+        if '.flv?' in url:
+            return 'flv', 'flv'
+        if 'hd.mp4?' in url:
+            return 'hdmp4', 'mp4'
+        if '.mp4?' in url:
+            return 'mp4', 'mp4'
+        raise Exception('Unknown stream type')
+
+    def api_req(self, cid, quality, bangumi):
+        ts = str(int(time.time()))
+        if not bangumi:
+            params_str = 'cid={}&player=1&quality={}&ts={}'.format(cid, quality, ts)
+            chksum = hashlib.md5(bytes(params_str+self.SEC1, 'utf8')).hexdigest()
+            api_url = self.api_url + params_str + '&sign=' + chksum
+        else:
+            params_str = 'cid={}&module=bangumi&player=1&quality={}&ts={}'.format(cid, quality, ts)
+            chksum = hashlib.md5(bytes(params_str+self.SEC2, 'utf8')).hexdigest()
+            api_url = self.bangumi_api_url + params_str + '&sign=' + chksum
+
+        xml_str = get_content(api_url)
+        return xml_str
+
+    def parse_bili_xml(self, xml_str):
+        urls_list = []
+        total_size = 0
+        doc = parseString(xml_str.encode('utf8'))
+        durls = doc.getElementsByTagName('durl')
+        for durl in durls:
+            size = durl.getElementsByTagName('size')[0]
+            total_size += int(size.firstChild.nodeValue)
+            url = durl.getElementsByTagName('url')[0]
+            urls_list.append(url.firstChild.nodeValue)
+        stream_type, container = self.bilibili_stream_type(urls_list)
+        if stream_type not in self.streams:
+            self.streams[stream_type] = {}
+            self.streams[stream_type]['src'] = urls_list
+            self.streams[stream_type]['size'] = total_size
+            self.streams[stream_type]['container'] = container
+
+    def download_by_vid(self, cid, bangumi, **kwargs):
+        stream_id = kwargs.get('stream_id')
+# guard here. if stream_id invalid, fallback as not stream_id
+        if stream_id and stream_id in self.fmt2qlt:
+            quality = stream_id
+        else:
+            quality = 'hdflv' if bangumi else 'flv'
+
+        info_only = kwargs.get('info_only')
+        if not info_only or stream_id:
+# won't be None
+            qlt = self.fmt2qlt.get(quality)
+            api_xml = self.api_req(cid, qlt, bangumi)
+            self.parse_bili_xml(api_xml)
+            self.danmuku = get_danmuku_xml(cid)
+        else:
+            for qlt in range(4, 0, -1):
+                api_xml = self.api_req(cid, qlt, bangumi)
+                self.parse_bili_xml(api_xml)
+
+    def prepare(self, **kwargs):
+        self.ua = fake_headers['User-Agent']
+        self.url = url_locations([self.url])[0]
+        frag = urllib.parse.urlparse(self.url).fragment
+# http://www.bilibili.com/video/av3141144/index_2.html#page=3
+        if frag:
+            hit = re.search(r'page=(\d+)', frag)
+            if hit is not None:
+                page = hit.group(1)
+                aid = re.search(r'av(\d+)', self.url).group(1)
+                self.url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, page)
+        self.referer = self.url
+        self.page = get_content(self.url)
+        self.title = re.search(r'<h1\s*title="([^"]+)"', self.page).group(1)
+        if 'subtitle' in kwargs:
+            subtitle = kwargs['subtitle']
+            self.title = '{} {}'.format(self.title, subtitle)
+
+        if 'bangumi.bilibili.com' in self.url:
+            self.bangumi_entry(**kwargs)
+        elif 'live.bilibili.com' in self.url:
+            self.live_entry(**kwargs)
+        else:
+            self.entry(**kwargs)
+
+    def entry(self, **kwargs):
+# tencent player
+        tc_flashvars = re.search(r'"bili-cid=\d+&bili-aid=\d+&vid=([^"]+)"', self.page)
+        if tc_flashvars:
+            tc_flashvars = tc_flashvars.group(1)
+        if tc_flashvars is not None:
+            self.out = True
+            qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
+            return
 
-    pool = int(pool)
-    assert 0 <= pool <= 2
-    # pool 0: normal
-    # pool 1: srt
-    # pool 2: special?
+        cid = re.search(r'cid=(\d+)', self.page).group(1)
+        if cid is not None:
+            self.download_by_vid(cid, False, **kwargs)
+        else:
+# flashvars?
+            flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)
+            if flashvars is None:
+                raise Exception('Unsupported page {}'.format(self.url))
+            param = flashvars.split('&')[0]
+            t, cid = param.split('=')
+            t = t.strip()
+            cid = cid.strip()
+            if t == 'vid':
+                sina_download_by_vid(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
+            elif t == 'ykid':
+                youku_download_by_vid(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
+            elif t == 'uid':
+                tudou_download_by_id(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
+            else:
+                raise NotImplementedError('Unknown flashvars {}'.format(flashvars))
+            return
 
-    font_size = int(font_size)
+    def live_entry(self, **kwargs):
+        self.title = re.search(r'<title>([^<]+)', self.page).group(1)
+        self.room_id = re.search('ROOMID\s*=\s*(\d+)', self.page).group(1)
+        api_url = self.live_api.format(self.room_id)
+        json_data = json.loads(get_content(api_url))
+        urls = [json_data['durl'][0]['url']]
+
+        self.streams['live'] = {}
+        self.streams['live']['src'] = urls
+        self.streams['live']['container'] = 'flv'
+        self.streams['live']['size'] = 0
+
+    def bangumi_entry(self, **kwargs):
+        bangumi_id = re.search(r'(\d+)', self.url).group(1)
+        bangumi_data = get_bangumi_info(bangumi_id)
+        bangumi_payment = bangumi_data.get('payment')
+        if bangumi_payment and bangumi_payment['price'] != '0':
+            log.w("It's a paid item")
+        ep_ids = collect_bangumi_epids(bangumi_data)
+
+        frag = urllib.parse.urlparse(self.url).fragment
+        if frag:
+            episode_id = frag
+        else:
+            episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page)
+        cont = post_content('http://bangumi.bilibili.com/web_api/get_source', post_data=dict(episode_id=episode_id))
+        cid = json.loads(cont)['result']['cid']
+        cont = get_content('http://bangumi.bilibili.com/web_api/episode/{}.json'.format(episode_id))
+        ep_info = json.loads(cont)['result']['currentEpisode']
 
-    font_color = '#%06x' % int(font_color)
+        long_title = ep_info['longTitle']
+        aid = ep_info['avId']
 
-    return pool, mode, font_size, font_color
+        idx = 0
+        while ep_ids[idx] != episode_id:
+            idx += 1
 
+        self.title = '{} [{} {}]'.format(self.title, idx+1, long_title)
+        self.download_by_vid(cid, bangumi=True, **kwargs)
 
-def parse_srt_xml(xml):
-    d = re.findall(r'<d p="([^"]+)">(.*)</d>', xml)
-    for x, y in d:
-        p = parse_srt_p(x)
-    raise NotImplementedError()
 
+def check_oversea():
+    url = 'https://interface.bilibili.com/player?id=cid:17778881'
+    xml_lines = get_content(url).split('\n')
+    for line in xml_lines:
+        key = line.split('>')[0][1:]
+        if key == 'country':
+            value = line.split('>')[1].split('<')[0]
+            if value != '中国':
+                return True
+            else:
+                return False
+    return False
+
+def check_sid():
+    if not cookies:
+        return False
+    for cookie in cookies:
+        if cookie.domain == '.bilibili.com' and cookie.name == 'sid':
+            return True
+    return False
+
+def fetch_sid(cid, aid):
+    url = 'http://interface.bilibili.com/player?id=cid:{}&aid={}'.format(cid, aid)
+    cookies = http.cookiejar.CookieJar()
+    req = urllib.request.Request(url)
+    res = urllib.request.urlopen(url)
+    cookies.extract_cookies(res, req)
+    for c in cookies:
+        if c.domain == '.bilibili.com' and c.name == 'sid':
+            return c.value
+    raise
+
+def collect_bangumi_epids(json_data):
+    eps = json_data['result']['episodes']
+    eps = sorted(eps, key=lambda item: int(item['index']))
+    result = []
+    for ep in eps:
+        result.append(ep['episode_id'])
+    return result
+
+def get_bangumi_info(bangumi_id):
+    BASE_URL = 'http://bangumi.bilibili.com/jsonp/seasoninfo/'
+    long_epoch = int(time.time() * 1000)
+    req_url = BASE_URL + bangumi_id + '.ver?callback=seasonListCallback&jsonp=jsonp&_=' + str(long_epoch)
+    season_data = get_content(req_url)
+    season_data = season_data[len('seasonListCallback('):]
+    season_data = season_data[: -1 * len(');')]
+    json_data = json.loads(season_data)
+    return json_data
+
+def get_danmuku_xml(cid):
+    return get_content('http://comment.bilibili.com/{}.xml'.format(cid))
 
 def parse_cid_playurl(xml):
     from xml.dom.minidom import parseString
     try:
+        urls_list = []
+        total_size = 0
         doc = parseString(xml.encode('utf-8'))
-        urls = [durl.getElementsByTagName('url')[0].firstChild.nodeValue for durl in doc.getElementsByTagName('durl')]
-        return urls
-    except:
-        return []
-
-
-def bilibili_download_by_cids(cids, title, output_dir='.', merge=True, info_only=False):
-    urls = []
-    for cid in cids:
-        sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
-        url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
-        urls += [i
-                 if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                 else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                 for i in parse_cid_playurl(get_content(url))]
-
-    type_ = ''
-    size = 0
-    for url in urls:
-        _, type_, temp = url_info(url)
-        size += temp
-
-    print_info(site_info, title, type_, size)
-    if not info_only:
-        download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, headers={'Referer': 'http://www.bilibili.com/'})
-
-
-def bilibili_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    while True:
-        try:
-            sign_this = hashlib.md5(bytes('cid={cid}&from=miniplay&player=1{SECRETKEY_MINILOADER}'.format(cid = cid, SECRETKEY_MINILOADER = SECRETKEY_MINILOADER), 'utf-8')).hexdigest()
-            url = 'http://interface.bilibili.com/playurl?&cid=' + cid + '&from=miniplay&player=1' + '&sign=' + sign_this
-            urls = [i
-                    if not re.match(r'.*\.qqvideo\.tc\.qq\.com', i)
-                    else re.sub(r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', i)
-                    for i in parse_cid_playurl(get_content(url))]
-
-            type_ = ''
-            size = 0
-            for url in urls:
-                _, type_, temp = url_info(url, headers={'Referer': 'http://www.bilibili.com/'})
-                size += temp or 0
-
-            print_info(site_info, title, type_, size)
-            if not info_only:
-                download_urls(urls, title, type_, total_size=None, output_dir=output_dir, merge=merge, timeout=1, headers={'Referer': 'http://www.bilibili.com/'})
-        except socket.timeout:
-            continue
-        else:
-            break
-
-
-def bilibili_live_download_by_cid(cid, title, output_dir='.', merge=True, info_only=False):
-    api_url = 'http://live.bilibili.com/api/playurl?cid=' + cid
-    urls = parse_cid_playurl(get_content(api_url))
-
-    for url in urls:
-        _, type_, _ = url_info(url)
-        size = 0
-        print_info(site_info, title, type_, size)
-        if not info_only:
-            download_urls([url], title, type_, total_size=None, output_dir=output_dir, merge=merge)
-
-
-def bilibili_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_content(url)
-
-    title = r1_of([r'<meta name="title" content="\s*([^<>]{1,999})\s*" />',
-                   r'<h1[^>]*>\s*([^<>]+)\s*</h1>'], html)
-    if title:
-        title = unescape_html(title)
-        title = escape_file_path(title)
-
-    if re.match(r'https?://bangumi\.bilibili\.com/', url):
-        # quick hack for bangumi URLs
-        episode_id = r1(r'#(\d+)$', url) or r1(r'first_ep_id = "(\d+)"', html)
-        cont = post_content('http://bangumi.bilibili.com/web_api/get_source',
-                            post_data={'episode_id': episode_id})
-        cid = json.loads(cont)['result']['cid']
-        title = '%s [%s]' % (title, episode_id)
-        bilibili_download_by_cid(str(cid), title, output_dir=output_dir, merge=merge, info_only=info_only)
-
+        durls = doc.getElementsByTagName('durl')
+        cdn_cnt = len(durls[0].getElementsByTagName('url'))
+        for i in range(cdn_cnt):
+            urls_list.append([])
+        for durl in durls:
+            size = durl.getElementsByTagName('size')[0]
+            total_size += int(size.firstChild.nodeValue)
+            cnt = len(durl.getElementsByTagName('url'))
+            for i in range(cnt):
+                u = durl.getElementsByTagName('url')[i].firstChild.nodeValue
+                urls_list[i].append(u)
+        return urls_list, total_size
+    except Exception as e:
+        log.w(e)
+        return [], 0
+
+def bilibili_download_playlist_by_url(url, **kwargs):
+    url = url_locations([url])[0]
+    if 'live.bilibili' in url:
+        site.download_by_url(url)
+    elif 'bangumi.bilibili' in url:
+        bangumi_id = re.search(r'(\d+)', url).group(1)
+        bangumi_data = get_bangumi_info(bangumi_id)
+        ep_ids = collect_bangumi_epids(bangumi_data)
+
+        base_url = url.split('#')[0]
+        for ep_id in ep_ids:
+            ep_url = '#'.join([base_url, ep_id])
+            Bilibili().download_by_url(ep_url, **kwargs)
     else:
-        flashvars = r1_of([r'(cid=\d+)', r'(cid: \d+)', r'flashvars="([^"]+)"',
-                           r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-        assert flashvars
-        flashvars = flashvars.replace(': ', '=')
-        t, cid = flashvars.split('=', 1)
-        cid = cid.split('&')[0]
-        if t == 'cid':
-            if re.match(r'https?://live\.bilibili\.com/', url):
-                title = r1(r'<title>\s*([^<>]+)\s*</title>', html)
-                bilibili_live_download_by_cid(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-
-            else:
-                # multi-P
-                cids = []
-                pages = re.findall('<option value=\'([^\']*)\'', html)
-                titles = re.findall('<option value=.*>\s*([^<>]+)\s*</option>', html)
-                for i, page in enumerate(pages):
-                    html = get_html("http://www.bilibili.com%s" % page)
-                    flashvars = r1_of([r'(cid=\d+)',
-                                       r'flashvars="([^"]+)"',
-                                       r'"https://[a-z]+\.bilibili\.com/secure,(cid=\d+)(?:&aid=\d+)?"'], html)
-                    if flashvars:
-                        t, cid = flashvars.split('=', 1)
-                        cids.append(cid.split('&')[0])
-                    if url.endswith(page):
-                        cids = [cid.split('&')[0]]
-                        titles = [titles[i]]
-                        break
-
-                # no multi-P
-                if not pages:
-                    cids = [cid]
-                    titles = [r1(r'<option value=.* selected>\s*([^<>]+)\s*</option>', html) or title]
-                for i in range(len(cids)):
-                    completeTitle=None
-                    if (title == titles[i]):
-                        completeTitle=title
-                    else:
-                        completeTitle=title+"-"+titles[i]#Build Better Title
-                    bilibili_download_by_cid(cids[i],
-                                             completeTitle,
-                                             output_dir=output_dir,
-                                             merge=merge,
-                                             info_only=info_only)
-
-        elif t == 'vid':
-            sina_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-        elif t == 'ykid':
-            youku_download_by_vid(cid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
-        elif t == 'uid':
-            tudou_download_by_id(cid, title, output_dir=output_dir, merge=merge, info_only=info_only)
-        else:
-            raise NotImplementedError(flashvars)
-
-    if not info_only and not dry_run:
-        if not kwargs['caption']:
-            print('Skipping danmaku.')
-            return
-        title = get_filename(title)
-        print('Downloading %s ...\n' % (title + '.cmt.xml'))
-        xml = get_srt_xml(cid)
-        with open(os.path.join(output_dir, title + '.cmt.xml'), 'w', encoding='utf-8') as x:
-            x.write(xml)
-
-
-site_info = "bilibili.com"
-download = bilibili_download
-download_playlist = bilibili_download
+        aid = re.search(r'av(\d+)', url).group(1)
+        page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
+        page_cnt = len(page_list)
+        for no in range(1, page_cnt+1):
+            page_url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, no)
+            subtitle = page_list[no-1]['pagename']
+            Bilibili().download_by_url(page_url, subtitle=subtitle, **kwargs)
+
+site = Bilibili()
+download = site.download_by_url
+download_playlist = bilibili_download_playlist_by_url
+
+bilibili_download = download

From 9d0c776dce928551c0d626f362aad285855ba516 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 31 May 2017 12:21:15 +0800
Subject: [PATCH 0390/1225] [youku]fix #2048

---
 src/you_get/extractors/youku.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 845a0b42c0..3cc678a93e 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -177,8 +177,11 @@ def prepare(self, **kwargs):
             data = youku_ups(self.vid, '0402')['data']
         else:
             data = youku_ups(self.vid)['data']
-        if data.get('error'):
-            log.wtf(data['error']['note'])
+        if data.get('stream') is None:
+            if data.get('error'):
+                log.wtf(data['error']['note'])
+            log.wtf('Unknown error')
+
         self.title = data['video']['title']
         stream_types = dict([(i['id'], i) for i in self.stream_types])
         audio_lang = data['stream'][0]['audio_lang']

From 1a3752781aa97ba8571fc1fd5b9a6c13c10a130e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 2 Jun 2017 16:09:19 +0800
Subject: [PATCH 0391/1225] [youku]hacks to quote cna

---
 src/you_get/extractors/youku.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 3cc678a93e..f67fdc39f5 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -10,16 +10,23 @@
 import traceback
 import json
 import urllib.request
+import urllib.parse
+
+def quote_cna(cna):
+    if '%' in cna:
+        return cna
+    return urllib.parse.quote(cna)
 
 def fetch_cna():
     if cookies:
         for cookie in cookies:
             if cookie.name == 'cna' and cookie.domain == '.youku.com':
                 log.i('Found cna in imported cookies. Use it')
-                return cookie.value
+                return quote_cna(cookie.value)
     url = 'http://gm.mmstat.com/yt/ykcomment.play.commentInit?cna='
     req = urllib.request.urlopen(url)
-    return req.info()['Set-Cookie'].split(';')[0].split('=')[1]
+    cna = req.info()['Set-Cookie'].split(';')[0].split('=')[1]
+    return quote_cna(cna)
 
 def youku_ups(vid, ccode='0401'):
     url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)

From 525f052332652157f2b44a0a7a5a7179616858fb Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 3 Jun 2017 10:12:12 +0800
Subject: [PATCH 0392/1225] [youku]cna hacks

---
 src/you_get/extractors/youku.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index f67fdc39f5..e0072e76cb 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -23,10 +23,17 @@ def fetch_cna():
             if cookie.name == 'cna' and cookie.domain == '.youku.com':
                 log.i('Found cna in imported cookies. Use it')
                 return quote_cna(cookie.value)
-    url = 'http://gm.mmstat.com/yt/ykcomment.play.commentInit?cna='
+    url = 'http://log.mmstat.com/eg.js'
     req = urllib.request.urlopen(url)
-    cna = req.info()['Set-Cookie'].split(';')[0].split('=')[1]
-    return quote_cna(cna)
+    headers = req.getheaders()
+    for header in headers:
+        if header[0].lower() == 'set-cookie':
+            n_v = header[1].split(';')[0]
+            name, value = n_v.split('=')
+            if name == 'cna':
+                return quote_cna(value)
+    log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
+    return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')
 
 def youku_ups(vid, ccode='0401'):
     url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)

From 9ca5bf1895bbd427f579291f77c7faddd0ddfcb1 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 6 Jul 2017 17:03:01 +0800
Subject: [PATCH 0393/1225] [bilibili] support free bangumi.bilibili.com/movie

---
 src/you_get/extractors/bilibili.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 8b18eeabc4..f904ea49a7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -114,18 +114,30 @@ def prepare(self, **kwargs):
                 self.url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, page)
         self.referer = self.url
         self.page = get_content(self.url)
-        self.title = re.search(r'<h1\s*title="([^"]+)"', self.page).group(1)
-        if 'subtitle' in kwargs:
-            subtitle = kwargs['subtitle']
-            self.title = '{} {}'.format(self.title, subtitle)
-
-        if 'bangumi.bilibili.com' in self.url:
+        try:
+            self.title = re.search(r'<h1\s*title="([^"]+)"', self.page).group(1)
+            if 'subtitle' in kwargs:
+                subtitle = kwargs['subtitle']
+                self.title = '{} {}'.format(self.title, subtitle)
+        except Exception:
+            pass
+        if 'bangumi.bilibili.com/movie' in self.url:
+            self.movie_entry(**kwargs)
+        elif 'bangumi.bilibili.com' in self.url:
             self.bangumi_entry(**kwargs)
         elif 'live.bilibili.com' in self.url:
             self.live_entry(**kwargs)
         else:
             self.entry(**kwargs)
 
+    def movie_entry(self, **kwargs):
+        patt = r"var\s*aid\s*=\s*'(\d+)'"
+        aid = re.search(patt, self.page).group(1)
+        page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
+        self.title = page_list[0]['pagename']
+# False for is_bangumi, old interface works for all free items
+        self.download_by_vid(page_list[0]['cid'], False, **kwargs)
+
     def entry(self, **kwargs):
 # tencent player
         tc_flashvars = re.search(r'"bili-cid=\d+&bili-aid=\d+&vid=([^"]+)"', self.page)
@@ -276,6 +288,7 @@ def parse_cid_playurl(xml):
 
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url])[0]
+# a bangumi here? possible?
     if 'live.bilibili' in url:
         site.download_by_url(url)
     elif 'bangumi.bilibili' in url:

From b885c833b1bbfe52d1eb05c6e41b8d34f40a2a1f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 7 Jul 2017 16:40:19 +0200
Subject: [PATCH 0394/1225] [youtube] VEVO: get over speed limiting (close
 #2068)

---
 src/you_get/extractors/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index d08c848ac6..4c9b31f038 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -328,6 +328,8 @@ def prepare(self, **kwargs):
                                   parse.unquote(i.split('=')[1]))
                                  for i in afmt.split('&')])
                            for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
+                for stream in streams: # get over speed limiting
+                    stream['url'] += '&ratebypass=yes'
                 for stream in streams: # audio
                     if stream['type'].startswith('audio/mp4'):
                         dash_mp4_a_url = stream['url']

From 23dbe2d07bd2edd41bbe1fd16e691252f1311728 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 01:35:16 +0200
Subject: [PATCH 0395/1225] [youtube] fix caption tracks extraction (close
 #2123)

---
 src/you_get/extractors/youtube.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 8335da9ebb..986906d6ec 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -224,14 +224,10 @@ def prepare(self, **kwargs):
 
         # Prepare caption tracks
         try:
-            caption_tracks = ytplayer_config['args']['caption_tracks'].split(',')
+            caption_tracks = json.loads(ytplayer_config['args']['player_response'])['captions']['playerCaptionsTracklistRenderer']['captionTracks']
             for ct in caption_tracks:
-                lang = None
-                for i in ct.split('&'):
-                    [k, v] = i.split('=')
-                    if k == 'lc' and lang is None: lang = v
-                    if k == 'v' and v[0] != '.': lang = v # auto-generated
-                    if k == 'u': ttsurl = parse.unquote_plus(v)
+                ttsurl, lang = ct['baseUrl'], ct['languageCode']
+
                 tts_xml = parseString(get_content(ttsurl))
                 transcript = tts_xml.getElementsByTagName('transcript')[0]
                 texts = transcript.getElementsByTagName('text')

From 72eb5cc34d5786ffb584fe18d25a4549c55de471 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 01:54:49 +0200
Subject: [PATCH 0396/1225] [instagram] support multiple image posts

---
 src/you_get/extractors/instagram.py | 40 +++++++++++++++++++++++------
 1 file changed, 32 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 907eda2cb6..3f488f9e3f 100644
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -10,19 +10,43 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     vid = r1(r'instagram.com/p/([^/]+)', url)
     description = r1(r'<meta property="og:title" content="([^"]*)"', html)
     title = "{} [{}]".format(description.replace("\n", " "), vid)
-
     stream = r1(r'<meta property="og:video" content="([^"]*)"', html)
     if stream:
         _, ext, size = url_info(stream)
+
+        print_info(site_info, title, ext, size)
+        if not info_only:
+            download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
-        image = r1(r'<meta property="og:image" content="([^"]*)"', html)
-        ext = 'jpg'
-        _, _, size = url_info(image)
+        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
+        info = json.loads(data.group(1))
 
-    print_info(site_info, title, ext, size)
-    url = stream if stream else image
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge=merge)
+        if 'edge_sidecar_to_children' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
+            edges = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['edge_sidecar_to_children']['edges']
+            for edge in edges:
+                title = edge['node']['shortcode']
+                image_url = edge['node']['display_url']
+                ext = image_url.split('.')[-1]
+                size = int(get_head(image_url)['Content-Length'])
+                print_info(site_info, title, ext, size)
+                if not info_only:
+                    download_urls(urls=[image_url],
+                                  title=title,
+                                  ext=ext,
+                                  total_size=size,
+                                  output_dir='.')
+        else:
+            title = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['shortcode']
+            image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
+            ext = image_url.split('.')[-1]
+            size = int(get_head(image_url)['Content-Length'])
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls(urls=[image_url],
+                              title=title,
+                              ext=ext,
+                              total_size=size,
+                              output_dir='.')
 
 site_info = "Instagram.com"
 download = instagram_download

From 490514eae6e19abc0280c2f525b7d432602e5a38 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 02:56:54 +0200
Subject: [PATCH 0397/1225] [universal] support HLS m3u8

---
 src/you_get/extractors/universal.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index a4262f618d..3168c48c6b 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -29,6 +29,17 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if page_title:
             page_title = unescape_html(page_title)
 
+        hls_urls = re.findall(r'(https?://[^;"\'\\]+' + '\.m3u8?' +
+                              r'[^;"\'\\]*)', page)
+        if hls_urls:
+            for hls_url in hls_urls:
+                type_, ext, size = url_info(hls_url)
+                print_info(site_info, page_title, type_, size)
+                if not info_only:
+                    download_url_ffmpeg(url=hls_url, title=page_title,
+                                        ext='mp4', output_dir=output_dir)
+            return
+
         # most common media file extensions on the Internet
         media_exts = ['\.flv', '\.mp3', '\.mp4', '\.webm',
                       '[-_]1\d\d\d\.jpe?g', '[-_][6-9]\d\d\.jpe?g', # tumblr

From bfdc08e6a1b2654316a506f35a182b6f956a31f5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 03:12:22 +0200
Subject: [PATCH 0398/1225] [google+] download original photos

---
 src/you_get/extractors/google.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 1f2c354c6b..c4e1a3f21a 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -60,6 +60,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
             real_urls.append(u)
         if not real_urls:
             real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
+            real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
         post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id

From 58fe436d0f396214bb7cd4dcfe836be89671a267 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 03:14:26 +0200
Subject: [PATCH 0399/1225] version 0.4.775

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 23eaeb29da..9f65a0e31c 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.750'
+__version__ = '0.4.775'

From 94121ea76d182e17150db1fca13eb2513c4ef04a Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 8 Jul 2017 18:22:35 +0800
Subject: [PATCH 0400/1225] [bilibili]fix non-integer index

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f904ea49a7..d22b7587eb 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -245,7 +245,7 @@ def fetch_sid(cid, aid):
 
 def collect_bangumi_epids(json_data):
     eps = json_data['result']['episodes']
-    eps = sorted(eps, key=lambda item: int(item['index']))
+    eps = sorted(eps, key=lambda item: float(item['index']))
     result = []
     for ep in eps:
         result.append(ep['episode_id'])

From fc4f35ae4d3351ad7d8061f74ed7bcc2e9729163 Mon Sep 17 00:00:00 2001
From: YenvY <dusk.embrace@gmail.com>
Date: Sat, 8 Jul 2017 19:59:24 +0800
Subject: [PATCH 0401/1225] fix apikey matching error in gallery case

---
 src/you_get/extractors/flickr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index f91d088415..4efa78ef45 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -77,7 +77,7 @@ def get_api_key(page):
     # since there's no place for a user to add custom infomation that may
     # misguide the regex in the homepage
     if not match:
-        return match1(get_html('https://flickr.com'), r'"site_key"\s*:\s*"([^"]+)"')
+        return match1(get_html('https://flickr.com'), pattern_inline_api_key)
     return match
 
 def get_NSID(url, page):

From d48a3a36e3efece1e02d5f294613aaadd21a4028 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Jul 2017 22:01:00 +0200
Subject: [PATCH 0402/1225] [youku] fix support of password-protected videos

---
 src/you_get/extractors/youku.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index e0072e76cb..1b03363455 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -35,11 +35,12 @@ def fetch_cna():
     log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
     return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')
 
-def youku_ups(vid, ccode='0401'):
+def youku_ups(vid, ccode='0401', password=None):
     url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)
     url += '&client_ip=192.168.1.1'
     url += '&utid=' + fetch_cna()
     url += '&client_ts=' + str(int(time.time()))
+    if password is not None: url += '&password=' + password
     return json.loads(get_content(url))
 
 class Youku(VideoExtractor):
@@ -193,8 +194,16 @@ def prepare(self, **kwargs):
             data = youku_ups(self.vid)['data']
         if data.get('stream') is None:
             if data.get('error'):
-                log.wtf(data['error']['note'])
-            log.wtf('Unknown error')
+                if data['error']['code'] == -2002:
+                    self.password_protected = True
+                    self.password = input(log.sprint('Password: ', log.YELLOW))
+                    data = youku_ups(self.vid, password=self.password)['data']
+                    if data.get('error'):
+                        log.wtf(data['error']['note'])
+                else:
+                    log.wtf(data['error']['note'])
+            else:
+                log.wtf('Unknown error')
 
         self.title = data['video']['title']
         stream_types = dict([(i['id'], i) for i in self.stream_types])

From ee506959511454f078586f3b10e96499e3eb9d2e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 9 Jul 2017 20:39:13 +0800
Subject: [PATCH 0403/1225] [vimeo]fix regex; do not panic if one id in a
 channel failed

---
 src/you_get/extractors/vimeo.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index 27c05b48f1..aa8db9444b 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -3,7 +3,10 @@
 __all__ = ['vimeo_download', 'vimeo_download_by_id', 'vimeo_download_by_channel', 'vimeo_download_by_channel_id']
 
 from ..common import *
+from ..util.log import *
 from json import loads
+import urllib.error
+
 access_token = 'f6785418277b72c7c87d3132c79eec24'  #By Beining
 
 #----------------------------------------------------------------------
@@ -25,7 +28,10 @@ def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_o
         id_list.append(match1(i['uri'], r'/videos/(\w+)'))
 
     for id in id_list:
-        vimeo_download_by_id(id, None, output_dir, merge, info_only)
+        try:
+            vimeo_download_by_id(id, None, output_dir, merge, info_only)
+        except urllib.error.URLError as e:
+            log.w('{} failed with {}'.format(id, e))
 
 def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
     try:
@@ -42,7 +48,7 @@ def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=F
 
         video_page = get_content('http://player.vimeo.com/video/%s' % id, headers=fake_headers)
         title = r1(r'<title>([^<]+)</title>', video_page)
-        info = loads(match1(video_page, r'var t=(\{[^;]+\});'))
+        info = loads(match1(video_page, r'var t=(\{.+?\});'))
 
     streams = info['request']['files']['progressive']
     streams = sorted(streams, key=lambda i: i['height'])

From dbe272a8af363544e7fc45a68aa86045e91d66be Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 9 Jul 2017 20:55:00 +0800
Subject: [PATCH 0404/1225] [common]revert #1940 for it breaks sohu

---
 src/you_get/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e98a80e901..9d688e0974 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -545,7 +545,8 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
             headers = headers
         else:
             headers = {}
-        headers['Range'] = 'bytes=' + str(received) + '-'
+        if received:
+            headers['Range'] = 'bytes=' + str(received) + '-'
         if refer:
             headers['Referer'] = refer
 

From 4891ae1783b5916727180a4958eba46e477195cc Mon Sep 17 00:00:00 2001
From: wwqgtxx <wwqgtxx@gmail.com>
Date: Mon, 10 Jul 2017 01:08:48 +0800
Subject: [PATCH 0405/1225] output refer and ua message in json

---
 src/you_get/json_output.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 0f0f48ec51..2c9950aed7 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -16,6 +16,13 @@ def output(video_extractor, pretty_print=True):
             out['audiolang'] = ve.audiolang
     except AttributeError:
         pass
+    extra = {}
+    if ve.referer is not None:
+        extra["referer"] = ve.referer
+    if ve.ua is not None:
+        extra["ua"] = ve.ua
+    if extra:
+        out["extra"] = extra
     if pretty_print:
         print(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))
     else:

From 6d96300c732a1a7051c8d9c118104dfcecb5997a Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 10 Jul 2017 15:51:27 +0800
Subject: [PATCH 0406/1225] [dailymotion]redirect to embed url to bypass
 restriction; prefer mp4

---
 src/you_get/extractors/dailymotion.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index 2e96c160ed..bbfd22fd5c 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -3,35 +3,36 @@
 __all__ = ['dailymotion_download']
 
 from ..common import *
+import urllib.parse
 
-def extract_m3u(url):
-    content = get_content(url)
-    m3u_url = re.findall(r'http://.*', content)[0]
-    return match1(m3u_url, r'([^#]+)')
+def rebuilt_url(url):
+    path = urllib.parse.urlparse(url).path
+    aid = path.split('/')[-1].split('_')[0]
+    return 'http://www.dailymotion.com/embed/video/{}?autoplay=1'.format(aid)
 
-def dailymotion_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+def dailymotion_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """Downloads Dailymotion videos by URL.
     """
 
-    html = get_content(url)
+    html = get_content(rebuilt_url(url))
     info = json.loads(match1(html, r'qualities":({.+?}),"'))
     title = match1(html, r'"video_title"\s*:\s*"([^"]+)"') or \
             match1(html, r'"title"\s*:\s*"([^"]+)"')
+    title = unicodize(title)
 
-    for quality in ['1080','720','480','380','240','auto']:
+    for quality in ['1080','720','480','380','240','144','auto']:
         try:
-            real_url = info[quality][0]["url"]
+            real_url = info[quality][1]["url"]
             if real_url:
                 break
         except KeyError:
             pass
 
-    m3u_url = extract_m3u(real_url)
-    mime, ext, size = 'video/mp4', 'mp4', 0
+    mime, ext, size = url_info(real_url)
 
     print_info(site_info, title, mime, size)
     if not info_only:
-        download_url_ffmpeg(m3u_url, title, ext, output_dir=output_dir, merge=merge)
+        download_urls(real_url, title, ext, output_dir=output_dir, merge=merge)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From 2fc0f49d58bc042a427336fb16ab59a923ef214b Mon Sep 17 00:00:00 2001
From: Hu Zhenyu <andyhuzhill@gmail.com>
Date: Wed, 12 Jul 2017 17:33:05 +0800
Subject: [PATCH 0407/1225]         fix get douyu real stream address

---
 src/you_get/extractors/douyutv.py | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 2f144ffa22..76055834f4 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -16,29 +16,23 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
     content = get_content(json_request_url)
-    data = json.loads(content)['data']
-    server_status = data.get('error',0)
+    json_content = json.loads(content)
+    data = json_content['data']
+    server_status = json_content.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
 
-    title = data.get('room_name')
-    show_status = data.get('show_status')
+    room_info_url = "http://open.douyucdn.cn/api/RoomApi/room/%s" % room_id
+    room_info_content = get_content(room_info_url)
+    room_info_obj = json.loads(room_info_content)
+    room_info_data = room_info_obj.get('data')
+
+    title = room_info_data.get('room_name')
+    show_status = room_info_data.get('room_status')
     if show_status is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
-    tt = int(time.time())
-    sign_content = 'lapi/live/thirdPart/getPlay/%s?aid=pcclient&rate=0&time=%s9TUk5fjjUjg9qIMH3sdnh' % (room_id, tt)
-    sign = hashlib.md5(sign_content.encode('ascii')).hexdigest()
-
-    json_request_url = "http://coapi.douyucdn.cn/lapi/live/thirdPart/getPlay/%s?rate=0" % room_id
-    headers = {'auth': sign, 'time': str(tt), 'aid': 'pcclient'}
-    content = get_content(json_request_url, headers = headers)
-    data = json.loads(content)['data']
-    server_status = data.get('error',0)
-    if server_status is not 0:
-        raise ValueError("Server returned error:%s" % server_status)
-
-    real_url = data.get('live_url')
+    real_url = data.get('hls_url')
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From e8624e63871b2924de4c3e66e67fdc9ba0f32de6 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 12 Jul 2017 20:14:40 +0800
Subject: [PATCH 0408/1225] [qq]fix crashes for preview segs

---
 src/you_get/extractors/qq.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index e7d5f146ca..4b4af01075 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -30,6 +30,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
+        if key_json.get('key') is None:
+            log.w(key_json['msg'])
+            break
         vkey = key_json['key']
         url = '{}{}?vkey={}'.format(host, filename, vkey)
         part_urls.append(url)

From 996634d4731c30ae35ff02ae08a39564316da03c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 12 Jul 2017 21:11:22 +0800
Subject: [PATCH 0409/1225] [qq]fix weixin patterns

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 4b4af01075..f103ed0048 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -95,7 +95,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
     if 'mp.weixin.qq.com/s?' in url:
         content = get_content(url)
-        vids = matchall(content, [r'\bvid=(\w+)'])
+        vids = matchall(content, [r'\?vid=(\w+)'])
         for vid in vids:
             qq_download_by_vid(vid, vid, output_dir, merge, info_only)
         return

From c48c4beb6bfca3f3997ca3f28b7d38bf8509d164 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 26 May 2017 12:17:29 +0800
Subject: [PATCH 0410/1225] [youku extractor]print m3u8 url in info

---
 src/you_get/extractor.py        | 4 ++++
 src/you_get/extractors/youku.py | 3 ++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 1a68dbafd3..8c7dcf7f70 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -22,6 +22,7 @@ def __init__(self, *args):
         self.url = None
         self.title = None
         self.vid = None
+        self.m3u8_url = None
         self.streams = {}
         self.streams_sorted = []
         self.audiolang = None
@@ -108,6 +109,9 @@ def p_stream(self, stream_id):
             if stream['size'] != float('inf')  and stream['size'] != 0:
                 print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
 
+        if 'm3u8_url' in stream:
+            print("      m3u8_url:      {}".format(stream['m3u8_url']))
+
         if 'itag' in stream:
             print("    # download-with: %s" % log.sprint("you-get --itag=%s [URL]" % stream_id, log.UNDERLINE))
         else:
diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 1b03363455..4ade977454 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -223,7 +223,8 @@ def prepare(self, **kwargs):
                         'size': stream['size'],
                         'pieces': [{
                             'segs': stream['segs']
-                        }]
+                        }],
+                        'm3u8_url': stream['m3u8_url']
                     }
                     src = []
                     for seg in stream['segs']:

From f47096c2016a6625abc5aaed2c32f9d82dd2984b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 20 Jun 2017 17:14:13 +0800
Subject: [PATCH 0411/1225] [iqiyi]H265 streams

---
 src/you_get/extractors/iqiyi.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 91329708a2..81dbef9513 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -97,7 +97,9 @@ class Iqiyi(VideoExtractor):
         {'id': '4k', 'container': 'm3u8', 'video_profile': '4k'},
         {'id': 'BD', 'container': 'm3u8', 'video_profile': '1080p'},
         {'id': 'TD', 'container': 'm3u8', 'video_profile': '720p'},
+        {'id': 'TD_H265', 'container': 'm3u8', 'video_profile': '720p H265'},
         {'id': 'HD', 'container': 'm3u8', 'video_profile': '540p'},
+        {'id': 'HD_H265', 'container': 'm3u8', 'video_profile': '540p H265'},
         {'id': 'SD', 'container': 'm3u8', 'video_profile': '360p'},
         {'id': 'LD', 'container': 'm3u8', 'video_profile': '210p'},
     ]
@@ -108,8 +110,8 @@ class Iqiyi(VideoExtractor):
     stream_to_bid = {  '4k': 10, 'fullhd' : 5, 'suprt-high' : 4, 'super' : 3, 'high' : 2, 'standard' :1, 'topspeed' :96}
     '''
     ids = ['4k','BD', 'TD', 'HD', 'SD', 'LD']
-    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD', 2: 'HD', 4: 'TD', 17: 'TD', 96: 'LD', 1: 'SD'}
-    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p'}
+    vd_2_id = {10: '4k', 19: '4k', 5:'BD', 18: 'BD', 21: 'HD_H265', 2: 'HD', 4: 'TD', 17: 'TD_H265', 96: 'LD', 1: 'SD', 14: 'TD'}
+    id_2_profile = {'4k':'4k', 'BD': '1080p','TD': '720p', 'HD': '540p', 'SD': '360p', 'LD': '210p', 'HD_H265': '540p H265', 'TD_H265': '720p H265'}
 
 
 
@@ -137,7 +139,7 @@ def prepare(self, **kwargs):
             self.title = match1(html, '<title>([^<]+)').split('-')[0]
         tvid, videoid = self.vid
         info = getVMS(tvid, videoid)
-        assert info['code'] == 'A00000', 'can\'t play this video'
+        assert info['code'] == 'A00000', "can't play this video"
 
         for stream in info['data']['vidl']:
             try:
@@ -145,8 +147,8 @@ def prepare(self, **kwargs):
                 if stream_id in self.stream_types:
                     continue
                 stream_profile = self.id_2_profile[stream_id]
-                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0}
-            except:
+                self.streams[stream_id] = {'video_profile': stream_profile, 'container': 'm3u8', 'src': [stream['m3u']], 'size' : 0, 'm3u8_url': stream['m3u']}
+            except Exception as e:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
     

From 1ab8ea015de66af75d090b0fd24403e80052122c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 15 Jul 2017 21:44:54 +0800
Subject: [PATCH 0412/1225] [common zhanqi]m3u8 code in common; rewrite zhanqi

---
 src/you_get/common.py            | 30 +++++++++--
 src/you_get/extractors/zhanqi.py | 85 +++++++++++++++-----------------
 2 files changed, 67 insertions(+), 48 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9d688e0974..2462bc851f 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -161,6 +161,22 @@ def rc4(key, data):
         out_list.append(char ^ prn)
 
     return bytes(out_list)
+
+def general_m3u8_extractor(url):
+    path_len = len(url.split('/')[-1])
+    base_url = url[:-path_len]
+
+    m3u8_list = get_content(url).split('\n')
+    urls = []
+    for line in m3u8_list:
+        line = line.strip()
+        if line and not line.startswith('#'):
+            if line.startswith('http'):
+                urls.append(line)
+            else:
+                urls.append(base_url + line)
+    return urls
+
 def maybe_print(*s):
     try: print(*s)
     except: pass
@@ -1027,7 +1043,7 @@ def f(*args, **kwargs):
         raise NotImplementedError('Playlist is not supported for ' + name)
     return f
 
-def print_info(site_info, title, type, size):
+def print_info(site_info, title, type, size, **kwargs):
     if json_output:
         json_output_.print_info(site_info=site_info, title=title, type=type, size=size)
         return
@@ -1092,14 +1108,22 @@ def print_info(site_info, title, type, size):
         type_info = "Portable Network Graphics (%s)" % type
     elif type in ['image/gif']:
         type_info = "Graphics Interchange Format (%s)" % type
-
+    elif type in ['m3u8']:
+        if 'm3u8_type' in kwargs:
+            if kwargs['m3u8_type'] == 'master':
+                type_info = 'M3U8 Master {}'.format(type)
+        else:
+            type_info = 'M3U8 Playlist {}'.format(type)
     else:
         type_info = "Unknown type (%s)" % type
 
     maybe_print("Site:      ", site_info)
     maybe_print("Title:     ", unescape_html(tr(title)))
     print("Type:      ", type_info)
-    print("Size:      ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
+    if type != 'm3u8':
+        print("Size:      ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
+    if type == 'm3u8' and 'm3u8_url' in kwargs:
+        print('M3U8 Url:   {}'.format(kwargs['m3u8_url']))
     print()
 
 def mime_to_container(mime):
diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index f2c673ca08..d0bbddb65a 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -4,53 +4,48 @@
 
 from ..common import *
 import json
+import base64
+from urllib.parse import urlparse
 
 def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    host_name = url.split('/')[2]
-    first_folder_path = url.split('/')[3].split('?')[0]
-
-    if first_folder_path != 'videos': #url = "https://www.zhanqi.tv/huashan?param_s=1_0.2.0"
-        if first_folder_path == 'topic': #https://www.zhanqi.tv/topic/lyingman
-            first_folder_path = url.split('/')[4].split('?')[0]
-        api_url = "https://www.zhanqi.tv/api/static/v2.1/room/domain/" + first_folder_path + ".json"
-        api_json = json.loads(get_html(api_url))
-        data = api_json['data']
-        status = data['status']
-        if status != '4':
-            raise ValueError ("The live stream is not online!")
-
-        nickname = data['nickname']
-        title = nickname + ": " + data['title']
-
-        roomid = data['id']
-        videoId = data['videoId']
-        jump_url = "http://wshdl.load.cdn.zhanqi.tv/zqlive/" + videoId + ".flv?get_url=1"
-        jump_url = jump_url.strip('\r\n')
-
-        real_url = get_html(jump_url)
-        real_url = real_url.strip('\r\n')
-        site_info = "www.zhanqi.tv"
-
-        print_info(site_info, title, 'flv', float('inf'))
-        if not info_only:
-            download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
+    path = urlparse(url).path[1:]
 
+    if not path.startswith('videos'): #url = "https://www.zhanqi.tv/huashan?param_s=1_0.2.0"
+        path_list = path.split('/')
+        room_id = path_list[1] if path_list[0] == 'topic' else path_list[0]
+        zhanqi_live(room_id, merge=merge, output_dir=output_dir, info_only=info_only, **kwargs)
     else: #url = 'https://www.zhanqi.tv/videos/Lyingman/2017/01/182308.html'
-        video_id = url.split('/')[-1].split('?')[0].split('.')[0]
-        assert video_id
-        api_url = "https://www.zhanqi.tv/api/static/v2.1/video/" + video_id + ".json"
-        api_json = json.loads(get_html(api_url))
-        data = api_json['data']
-
-        title = data['title']
-
-        video_url_id = data['flashvars']['VideoID']
-        real_url = "http://dlvod.cdn.zhanqi.tv/" + video_url_id
-        site_info = "www.zhanqi.tv/videos"
-
-        print_info(site_info, title, 'flv', float('inf'))
-        if not info_only:
-            download_url_ffmpeg(real_url, title, 'flv', {}, output_dir = output_dir, merge = merge)
-
+        video_id = path.split('.')[0].split('/')[-1]
+        zhanqi_video(video_id, merge=merge, output_dir=output_dir, info_only=info_only, **kwargs)
+
+def zhanqi_live(room_id, merge=True, output_dir='.', info_only=False, **kwargs):
+    api_url = "https://www.zhanqi.tv/api/static/v2.1/room/domain/{}.json".format(room_id)
+    json_data = json.loads(get_content(api_url))['data']
+    status = json_data['status']
+    if status != '4':
+        raise Exception("The live stream is not online!")
+
+    nickname = json_data['nickname']
+    title = nickname + ": " + json_data['title']
+    video_levels = base64.b64decode(json_data['flashvars']['VideoLevels']).decode('utf8')
+    m3u8_url = json.loads(video_levels)['streamUrl']
+
+    print_info(site_info, title, 'm3u8', 0, m3u8_url=m3u8_url, m3u8_type='master')
+    if not info_only:
+        download_url_ffmpeg(m3u8_url, title, 'mp4', output_dir=output_dir, merge=merge)
+
+def zhanqi_video(video_id, output_dir='.', info_only=False, merge=True, **kwargs):
+    api_url = 'https://www.zhanqi.tv/api/static/v2.1/video/{}.json'.format(video_id)
+    json_data = json.loads(get_content(api_url))['data']
+
+    title = json_data['title']
+    vid = json_data['flashvars']['VideoID']
+    m3u8_url = 'http://dlvod.cdn.zhanqi.tv/' + vid
+    urls = general_m3u8_extractor(m3u8_url)
+    print_info(site_info, title, 'm3u8', 0)
+    if not info_only:
+        download_urls(urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs)
+
+site_info = "www.zhanqi.tv"
 download = zhanqi_download
-download_playlist = playlist_not_supported('zhanqi')
\ No newline at end of file
+download_playlist = playlist_not_supported('zhanqi')

From 15c46d3f7d9ef45f48b95bc3fb96f1cb009b2b46 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 16 Jul 2017 08:49:42 +0800
Subject: [PATCH 0413/1225] [ffmpeg]drop -re flag when input is not live stream

---
 src/you_get/common.py           | 4 ++--
 src/you_get/extractors/iqiyi.py | 4 +---
 src/you_get/processor/ffmpeg.py | 7 +++++--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9d688e0974..64c791f2e9 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -997,7 +997,7 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re
     assert has_rtmpdump_installed(), "RTMPDump not installed."
     download_rtmpdump_stream(url,  title, ext,params, output_dir)
 
-def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
+def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False, stream=True):
     assert url
     if dry_run:
         print('Real URL:\n%s\n' % [url])
@@ -1020,7 +1020,7 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
 
     title = tr(get_filename(title))
 
-    ffmpeg_download_stream(url, title, ext, params, output_dir)
+    ffmpeg_download_stream(url, title, ext, params, output_dir, stream=stream)
 
 def playlist_not_supported(name):
     def f(*args, **kwargs):
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 5d636ec994..ee48942dec 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -206,9 +206,7 @@ def download(self, **kwargs):
             # For legacy main()
             
             #Here's the change!!
-            download_url_ffmpeg(urls[0], self.title, 'mp4',
-                          output_dir=kwargs['output_dir'],
-                          merge=kwargs['merge'],)
+            download_url_ffmpeg(urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], stream=False)
 
             if not kwargs['caption']:
                 print('Skipping captions.')
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c7631b600c..c7b362e103 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -207,7 +207,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         os.remove(file + '.ts')
     return True
 
-def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
+def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=True):
     """str, str->True
     WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
     You can basicly download anything with this function
@@ -219,7 +219,10 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.'):
         output = output_dir + '/' + output
 
     print('Downloading streaming content with FFmpeg, press q to stop recording...')
-    ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
+    if stream:
+        ffmpeg_params = [FFMPEG] + ['-y', '-re', '-i']
+    else:
+        ffmpeg_params = [FFMPEG] + ['-y', '-i']
     ffmpeg_params.append(files)  #not the same here!!!!
 
     if FFMPEG == 'avconv':  #who cares?

From 56efb0ba961bdedd109004339aba09b7d003a9ab Mon Sep 17 00:00:00 2001
From: wwqgtxx <wwqgtxx@gmail.com>
Date: Sun, 16 Jul 2017 23:50:53 +0800
Subject: [PATCH 0414/1225] add support for send the password from cli

---
 src/you_get/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 64c791f2e9..4e8e168f81 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1215,10 +1215,11 @@ def version():
     -t | --timeout <SECONDS>            Set socket timeout.
     -d | --debug                        Show traceback and other debug info.
     -I | --input-file                   Read non-playlist urls from file.
+    -P | --password <PASSWORD>          Set video visit password to PASSWORD.
     '''
 
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=', 'input-file=']
+    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:P:'
+    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=', 'input-file=', 'password=']
 #dead code? download_playlist is a function and always True
 #if download_playlist:
     short_opts = 'l' + short_opts
@@ -1252,6 +1253,7 @@ def version():
     traceback = False
     timeout = 600
     urls_from_file = []
+    password = None
 
     for o, a in opts:
         if o in ('-V', '--version'):
@@ -1330,6 +1332,8 @@ def version():
             lang = a
         elif o in ('-t', '--timeout'):
             timeout = int(a)
+        elif o in ('-P', '--password',):
+            password = a
         elif o in ('-I', '--input-file'):
             logging.debug('you are trying to load urls from {}'.format(a))
             if playlist:

From 5104e0443835bde398b7a6ab8f3d0b2c00c8035c Mon Sep 17 00:00:00 2001
From: wwqgtxx <wwqgtxx@gmail.com>
Date: Sun, 16 Jul 2017 23:53:11 +0800
Subject: [PATCH 0415/1225] get password from cli in youku.py

---
 src/you_get/extractors/youku.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 4ade977454..b83e473a9f 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -196,7 +196,9 @@ def prepare(self, **kwargs):
             if data.get('error'):
                 if data['error']['code'] == -2002:
                     self.password_protected = True
-                    self.password = input(log.sprint('Password: ', log.YELLOW))
+                    self.password = kwargs.get("password", None)
+                    if not self.password:
+                        self.password = input(log.sprint('Password: ', log.YELLOW))
                     data = youku_ups(self.vid, password=self.password)['data']
                     if data.get('error'):
                         log.wtf(data['error']['note'])

From 5bbb536de46db2ae004a390aa5352a1864459e6d Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 17 Jul 2017 16:17:43 +0800
Subject: [PATCH 0416/1225] fix weibo title when character "<" existed

---
 src/you_get/extractors/miaopai.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index e9a6ff2df2..dc52a25117 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -18,11 +18,14 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
 
     mobile_page = get_content(page_url, headers=fake_headers_mobile)
     url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
-    title = match1(mobile_page, r'<title>([^<]+)</title>')
+    title = match1(mobile_page, r'<title>((.|\n)+?)</title>')
+    if not title:
+        title = fid
+    title = title.replace('\n', '_')
     type_, ext, size = url_info(url)
-    print_info(site_info, title, type_, size)
+    print_info(site_info, title, 'mp4', size)
     if not info_only:
-        download_urls([url], title.replace('\n',''), ext, total_size=None, output_dir=output_dir, merge=merge)
+        download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
 #----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):

From 0f839bafac6aca836014ead88e7bfbca91a6d043 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 18 Jul 2017 09:16:01 +0800
Subject: [PATCH 0417/1225] [nicovideo]fix title

---
 src/you_get/extractors/nicovideo.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/nicovideo.py b/src/you_get/extractors/nicovideo.py
index 3df933da2d..da9129dec4 100644
--- a/src/you_get/extractors/nicovideo.py
+++ b/src/you_get/extractors/nicovideo.py
@@ -31,10 +31,11 @@ def nicovideo_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     nicovideo_login(user, password)
 
     html = get_html(url) # necessary!
-    title = unicodize(r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html))
+    title = r1(r'<title>(.+?)</title>', html)
+    #title = unicodize(r1(r'<span class="videoHeaderTitle"[^>]*>([^<]+)</span>', html))
 
     vid = url.split('/')[-1].split('?')[0]
-    api_html = get_html('http://www.nicovideo.jp/api/getflv?v=%s' % vid)
+    api_html = get_html('http://flapi.nicovideo.jp/api/getflv?v=%s' % vid)
     real_url = parse.unquote(r1(r'url=([^&]+)&', api_html))
 
     type, ext, size = url_info(real_url)

From 9b32634088a40576d18359480b401069cba03cde Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 20 Jul 2017 11:38:49 +0800
Subject: [PATCH 0418/1225] [acfun]endpoint url changed and checks referer

---
 src/you_get/extractors/acfun.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 95c66d80ac..e4f0b0d77a 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -13,14 +13,16 @@
 import json
 import re
 import base64
+import time
 
 def get_srt_json(id):
     url = 'http://danmu.aixifan.com/V2/%s' % id
     return get_content(url)
 
-def youku_acfun_proxy(vid, sign):
-    url = 'http://aplay-vod.cn-beijing.aliyuncs.com/acfun/web?vid={}&ct=85&ev=3&sign={}'.format(vid, sign)
-    json_data = json.loads(get_content(url))['data']
+def youku_acfun_proxy(vid, sign, ref):
+    endpoint = 'http://player.acfun.cn/flash_data?vid={}&ct=85&ev=3&sign={}&time={}'
+    url = endpoint.format(vid, sign, str(int(time.time() * 1000)))
+    json_data = json.loads(get_content(url, headers=dict(referer=ref)))['data']
     enc_text = base64.b64decode(json_data)
     dec_text = rc4(b'8bdc7e1a', enc_text).decode('utf8')
     youku_json = json.loads(dec_text)
@@ -70,7 +72,8 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
         #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
 #In Mar. 2017 there is a dedicated ``acfun_proxy'' in youku cloud player
 #old code removed
-        yk_streams = youku_acfun_proxy(info['sourceId'], info['encode'])
+        url = 'http://www.acfun.cn/v/ac' + vid
+        yk_streams = youku_acfun_proxy(info['sourceId'], info['encode'], url)
         seq = ['mp4hd3', 'mp4hd2', 'mp4hd', 'flvhd']
         for t in seq:
             if yk_streams.get(t):

From 6fa81497d539a99fda2636126df3e485179a4edd Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 20 Jul 2017 17:14:05 +0800
Subject: [PATCH 0419/1225] [youtube]use mp4_audio track when no audio track
 for webm

---
 src/you_get/extractors/youtube.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 986906d6ec..3b412dc24d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -366,14 +366,22 @@ def prepare(self, **kwargs):
                                 dash_url += '&signature={}'.format(sig)
                             dash_size = stream['clen']
                             itag = stream['itag']
+                            audio_url = None
+                            audio_size = None
+                            try:
+                                audio_url = dash_webm_a_url
+                                audio_size = int(dash_webm_a_size)
+                            except UnboundLocalError as e:
+                                audio_url = dash_mp4_a_url
+                                audio_size = int(dash_mp4_a_size)
                             self.dash_streams[itag] = {
                                 'quality': stream['size'],
                                 'itag': itag,
                                 'type': mimeType,
                                 'mime': mimeType,
                                 'container': 'webm',
-                                'src': [dash_url, dash_webm_a_url],
-                                'size': int(dash_size) + int(dash_webm_a_size)
+                                'src': [dash_url, audio_url],
+                                'size': int(dash_size) + int(audio_size)
                             }
 
     def extract(self, **kwargs):

From 6c0e4b796397a2b2ca2f26bbb865c03cce59b99a Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 20 Jul 2017 20:52:29 +0800
Subject: [PATCH 0420/1225] [youku]api endpoint now checks referer

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 4ade977454..c1db2de5fb 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -35,13 +35,13 @@ def fetch_cna():
     log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
     return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')
 
-def youku_ups(vid, ccode='0401', password=None):
+def youku_ups(vid, ccode='0401', password=None, referer='http://v.youku.com'):
     url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)
     url += '&client_ip=192.168.1.1'
     url += '&utid=' + fetch_cna()
     url += '&client_ts=' + str(int(time.time()))
     if password is not None: url += '&password=' + password
-    return json.loads(get_content(url))
+    return json.loads(get_content(url, headers=dict(Referer=referer)))
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"

From 57b432ff6403f9c47b91891f3382d9ef70ef4e5a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jul 2017 23:08:59 +0200
Subject: [PATCH 0421/1225] version 0.4.803

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 9f65a0e31c..b3de196f94 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.775'
+__version__ = '0.4.803'

From 728360f4ccc7c862425b5820dbae7ef4b0c4b38e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 24 Jul 2017 07:29:00 +0800
Subject: [PATCH 0422/1225] [util.log]fix #2177

---
 src/you_get/util/log.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
index b858789ccd..a2c77ab50b 100644
--- a/src/you_get/util/log.py
+++ b/src/you_get/util/log.py
@@ -89,10 +89,10 @@ def e(message, exit_code=None):
     """Print an error log message."""
     print_log(message, YELLOW, BOLD)
     if exit_code is not None:
-        exit(exit_code)
+        sys.exit(exit_code)
 
 def wtf(message, exit_code=1):
     """What a Terrible Failure!"""
     print_log(message, RED, BOLD)
     if exit_code is not None:
-        exit(exit_code)
+        sys.exit(exit_code)

From 0f835667bff997016e05400535d8fa292b4f2b84 Mon Sep 17 00:00:00 2001
From: kolen <incredible.angst@gmail.com>
Date: Mon, 24 Jul 2017 20:22:24 +0300
Subject: [PATCH 0423/1225] [instagram] Fix always downloading to current dir,
 ignoring `-o`

GH-2188
---
 src/you_get/extractors/instagram.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/instagram.py

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
old mode 100644
new mode 100755
index 3f488f9e3f..4b02ed71f1
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -34,7 +34,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                                   title=title,
                                   ext=ext,
                                   total_size=size,
-                                  output_dir='.')
+                                  output_dir=output_dir)
         else:
             title = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['shortcode']
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
@@ -46,7 +46,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                               title=title,
                               ext=ext,
                               total_size=size,
-                              output_dir='.')
+                              output_dir=output_dir)
 
 site_info = "Instagram.com"
 download = instagram_download

From e614cda733f76ab80ed008d51a14b4520a38daf3 Mon Sep 17 00:00:00 2001
From: wwqgtxx <wwqgtxx@gmail.com>
Date: Tue, 25 Jul 2017 21:49:23 +0800
Subject: [PATCH 0424/1225] avoid AttributeError in json_output.py

---
 src/you_get/json_output.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 2c9950aed7..d447ea2f56 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -17,9 +17,9 @@ def output(video_extractor, pretty_print=True):
     except AttributeError:
         pass
     extra = {}
-    if ve.referer is not None:
+    if getattr(ve, 'referer', None) is not None:
         extra["referer"] = ve.referer
-    if ve.ua is not None:
+    if getattr(ve, 'ua', None) is not None:
         extra["ua"] = ve.ua
     if extra:
         out["extra"] = extra

From 7eb7ead38020bb17683dc3e7c5e609b150771dad Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 29 Jul 2017 19:59:29 +0800
Subject: [PATCH 0425/1225] [soundcloud]update client id and new api

---
 src/you_get/common.py                |  5 ++++
 src/you_get/extractors/soundcloud.py | 40 ++++++++++++++++++++--------
 2 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 4e8e168f81..33c8acf67e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -459,6 +459,9 @@ def url_info(url, faker = False, headers = {}):
         'video/x-ms-asf': 'asf',
         'audio/mp4': 'mp4',
         'audio/mpeg': 'mp3',
+        'audio/wav': 'wav',
+        'audio/x-wav': 'wav',
+        'audio/wave': 'wav',
         'image/jpeg': 'jpg',
         'image/png': 'png',
         'image/gif': 'gif',
@@ -1085,6 +1088,8 @@ def print_info(site_info, title, type, size):
         type_info = "MPEG-4 audio (%s)" % type
     elif type in ['audio/mpeg']:
         type_info = "MP3 (%s)" % type
+    elif type in ['audio/wav', 'audio/wave', 'audio/x-wav']:
+        type_info = 'Waveform Audio File Format ({})'.format(type)
 
     elif type in ['image/jpeg']:
         type_info = "JPEG Image (%s)" % type
diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
index 97d9601225..6115041c8b 100644
--- a/src/you_get/extractors/soundcloud.py
+++ b/src/you_get/extractors/soundcloud.py
@@ -3,28 +3,46 @@
 __all__ = ['soundcloud_download', 'soundcloud_download_by_id']
 
 from ..common import *
+import json
+import urllib.error
 
-def soundcloud_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
+client_id = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
+
+def soundcloud_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False):
     assert title
+    url = 'https://api.soundcloud.com/tracks/{}/{}?client_id={}'.format(id, 'stream', client_id)
     
-    #if info["downloadable"]:
-    #   url = 'https://api.soundcloud.com/tracks/' + id + '/download?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
-    url = 'https://api.soundcloud.com/tracks/' + id + '/stream?client_id=02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea'
-    assert url
     type, ext, size = url_info(url)
     
     print_info(site_info, title, type, size)
+
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge = merge)
 
-def soundcloud_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    metadata = get_html('https://api.soundcloud.com/resolve.json?url=' + url + '&client_id=02gUJC0hH2ct1EGOcYXQIzRFU91c72Ea')
-    import json
+def soundcloud_i1_api(track_id):
+    url = 'https://api.soundcloud.com/i1/tracks/{}/streams?client_id={}'.format(track_id, client_id)
+    return json.loads(get_content(url))['http_mp3_128_url']
+
+def soundcloud_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    url = 'https://api.soundcloud.com/resolve.json?url={}&client_id={}'.format(url, client_id)
+    metadata = get_content(url)
     info = json.loads(metadata)
     title = info["title"]
-    id = str(info["id"])
-    
-    soundcloud_download_by_id(id, title, output_dir, merge = merge, info_only = info_only)
+    real_url = info.get('download_url')
+    if real_url is None:
+        real_url = info.get('steram_url')
+    if real_url is None:
+        raise Exception('Cannot get media URI for {}'.format(url))
+    real_url = '{}?client_id={}'.format(real_url, client_id)
+    try:
+        mime, ext, size = url_info(real_url)
+    except urllib.error.HTTPError as e:
+        if 401 == e.status:
+            real_url = soundcloud_i1_api(info['id'])
+            mime, ext, size = url_info(real_url)
+    print_info(site_info, title, mime, size)
+    if not info_only:
+        download_urls([real_url], title, ext, size, output_dir, merge=merge)
 
 site_info = "SoundCloud.com"
 download = soundcloud_download

From fca1ec8d9e4e446cfd274882ac299a5b45daf1ed Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 30 Jul 2017 22:15:24 +0800
Subject: [PATCH 0426/1225] [miaopai]new url pattern

---
 src/you_get/extractors/miaopai.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index dc52a25117..ca543d15bc 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -4,16 +4,18 @@
 
 from ..common import *
 import urllib.error
+import urllib.parse
+
+fake_headers_mobile = {
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Charset': 'UTF-8,*;q=0.5',
+    'Accept-Encoding': 'gzip,deflate,sdch',
+    'Accept-Language': 'en-US,en;q=0.8',
+    'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
+}
 
 def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    fake_headers_mobile = {
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Charset': 'UTF-8,*;q=0.5',
-        'Accept-Encoding': 'gzip,deflate,sdch',
-        'Accept-Language': 'en-US,en;q=0.8',
-        'User-Agent': 'Mozilla/5.0 (Linux; Android 4.4.2; Nexus 4 Build/KOT49H) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.114 Mobile Safari/537.36'
-    }
     page_url = 'http://video.weibo.com/show?fid=' + fid + '&type=mp4'
 
     mobile_page = get_content(page_url, headers=fake_headers_mobile)
@@ -36,7 +38,13 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
         fid = match1(url, r'/p/230444(\w+)')
         miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
     else:
-        raise Exception('Unknown pattern')
+        mobile_page = get_content(url, headers = fake_headers_mobile)
+        hit = re.search(r'"page_url"\s*:\s*"([^"]+)"', mobile_page)
+        if not hit:
+            raise Exception('Unknown pattern')
+        else:
+            escaped_url = hit.group(1)
+            miaopai_download(urllib.parse.unquote(escaped_url), output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
 site_info = "miaopai"
 download = miaopai_download

From c4c78721de34ea7bfbf69ed9e68cabfd847472ac Mon Sep 17 00:00:00 2001
From: Feilong Ma <mafeilong@gmail.com>
Date: Mon, 31 Jul 2017 05:26:00 -0400
Subject: [PATCH 0427/1225] Fix parsing irregular episode index

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index d22b7587eb..f78c2635fb 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -245,7 +245,7 @@ def fetch_sid(cid, aid):
 
 def collect_bangumi_epids(json_data):
     eps = json_data['result']['episodes']
-    eps = sorted(eps, key=lambda item: float(item['index']))
+    eps = sorted(eps, key=lambda item: float(item['index'].split('-')[0].split('+')[0]))
     result = []
     for ep in eps:
         result.append(ep['episode_id'])

From d4a12b403fa3ca8e7698544ff6dd6cb1067b6a71 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 4 Aug 2017 21:42:15 +0800
Subject: [PATCH 0428/1225] [baomihua]fix #2220

---
 src/you_get/extractors/baomihua.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
index 4c4febb71a..99dd7132ef 100644
--- a/src/you_get/extractors/baomihua.py
+++ b/src/you_get/extractors/baomihua.py
@@ -14,7 +14,8 @@ def baomihua_download_by_id(id, title=None, output_dir='.', merge=True, info_onl
     assert type
     vid = r1(r'&stream_name=([^&]*)', html)
     assert vid
-    url = "http://%s/pomoho_video/%s.%s" % (host, vid, type)
+    dir_str = r1(r'&dir=([^&]*)', html).strip()
+    url = "http://%s/%s/%s.%s" % (host, dir_str, vid, type)
     _, ext, size = url_info(url)
     print_info(site_info, title, type, size)
     if not info_only:

From 70c37bd272402ed86c8a404c431f32eb0f475498 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 5 Aug 2017 12:17:55 +0800
Subject: [PATCH 0429/1225] [ifeng]add one new url pattern

---
 src/you_get/extractors/ifeng.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ifeng.py b/src/you_get/extractors/ifeng.py
index b1b8524a2e..1c66f3879f 100644
--- a/src/you_get/extractors/ifeng.py
+++ b/src/you_get/extractors/ifeng.py
@@ -21,7 +21,9 @@ def ifeng_download_by_id(id, title = None, output_dir = '.', merge = True, info_
         download_urls([url], title, ext, size, output_dir, merge = merge)
 
 def ifeng_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    id = r1(r'/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.shtml$', url)
+# old pattern /uuid.shtml
+# now it could be #uuid
+    id = r1(r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})', url)
     if id:
         return ifeng_download_by_id(id, None, output_dir = output_dir, merge = merge, info_only = info_only)
 

From 4d0803bafb97467bbc47d050c6e2bede9069356a Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 5 Aug 2017 12:54:51 +0800
Subject: [PATCH 0430/1225] [ted]page js data changed

---
 src/you_get/extractors/ted.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/ted.py b/src/you_get/extractors/ted.py
index bb26baaa3c..c7dd87a16d 100644
--- a/src/you_get/extractors/ted.py
+++ b/src/you_get/extractors/ted.py
@@ -7,9 +7,10 @@
 
 def ted_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
-    metadata = json.loads(match1(html, r'({"talks"(.*)})\)'))
+    patt = r'"__INITIAL_DATA__"\s*:\s*\{(.+)\}'
+    metadata = json.loads('{' + match1(html, patt) + '}')
     title = metadata['talks'][0]['title']
-    nativeDownloads = metadata['talks'][0]['nativeDownloads']
+    nativeDownloads = metadata['talks'][0]['downloads']['nativeDownloads']
     for quality in ['high', 'medium', 'low']:
         if quality in nativeDownloads:
             url = nativeDownloads[quality]

From 50d22ee2250eee53c607733fd6d977a366c0ae71 Mon Sep 17 00:00:00 2001
From: YK Liu <cos.lyk@gmail.com>
Date: Sat, 5 Aug 2017 18:35:21 +0800
Subject: [PATCH 0431/1225] add new acfun's url pattern

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index e4f0b0d77a..87ead98ed6 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -105,7 +105,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
             pass
 
 def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    assert re.match(r'http://[^\.]+.acfun.[^\.]+/\D/\D\D(\d+)', url)
+    assert re.match(r'http://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url)
     html = get_content(url)
 
     title = r1(r'data-title="([^"]+)"', html)

From 6dd51760c1497155414fc40d427202a6e5c735bd Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 5 Aug 2017 20:46:32 +0800
Subject: [PATCH 0432/1225] [iqilu]page vars changed

---
 src/you_get/extractors/iqilu.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqilu.py b/src/you_get/extractors/iqilu.py
index 6c1657cf61..b6d47e2454 100644
--- a/src/you_get/extractors/iqilu.py
+++ b/src/you_get/extractors/iqilu.py
@@ -3,14 +3,18 @@
 __all__ = ['iqilu_download']
 
 from ..common import *
+import json
 
 def iqilu_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     ''''''
     if re.match(r'http://v.iqilu.com/\w+', url):
+        patt = r'url\s*:\s*\[([^\]]+)\]'
         
         #URL in webpage
         html = get_content(url)
-        url = match1(html, r"<input type='hidden' id='playerId' url='(.+)'")
+        player_data = '[' + match1(html, patt) + ']'
+        urls = json.loads(player_data)
+        url = urls[0]['stream_url']
         
         #grab title
         title = match1(html, r'<meta name="description" content="(.*?)\"\W')

From 1cd4ab7a1967e2cf475a6ec7784b0003da6236a1 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 5 Aug 2017 21:49:58 +0800
Subject: [PATCH 0433/1225] [bilibili]add support for vc.bilibili

---
 src/you_get/extractors/bilibili.py | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f78c2635fb..0cd217a069 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -33,7 +33,8 @@ class Bilibili(VideoExtractor):
             {'id': 'flv'},
             {'id': 'hdmp4'},
             {'id': 'mp4'},
-            {'id': 'live'}
+            {'id': 'live'},
+            {'id': 'vc'}
     ]
     fmt2qlt = dict(hdflv=4, flv=3, hdmp4=2, mp4=1)
 
@@ -127,6 +128,8 @@ def prepare(self, **kwargs):
             self.bangumi_entry(**kwargs)
         elif 'live.bilibili.com' in self.url:
             self.live_entry(**kwargs)
+        elif 'vc.bilibili.com' in self.url:
+            self.vc_entry(**kwargs)
         else:
             self.entry(**kwargs)
 
@@ -182,6 +185,24 @@ def live_entry(self, **kwargs):
         self.streams['live']['container'] = 'flv'
         self.streams['live']['size'] = 0
 
+    def vc_entry(self, **kwargs):
+        vc_id = re.search(r'video/(\d+)', self.url)
+        if not vc_id:
+            vc_id = re.search(r'vcdetail\?vc=(\d+)', self.url)
+            if not vc_id:
+                log.wtf('Unknown url pattern')
+        endpoint = 'http://api.vc.bilibili.com/clip/v1/video/detail?video_id={}&need_playurl=1'.format(vc_id.group(1))
+        vc_meta = json.loads(get_content(endpoint, headers=fake_headers))
+        if vc_meta['code'] != 0:
+            log.wtf('{}\n{}'.format(vc_meta['msg'], vc_meta['message']))
+        item = vc_meta['data']['item']
+        self.title = item['description']
+
+        self.streams['vc'] = {}
+        self.streams['vc']['src'] = [item['video_playurl']]
+        self.streams['vc']['container'] = 'mp4'
+        self.streams['vc']['size'] = int(item['video_size'])
+
     def bangumi_entry(self, **kwargs):
         bangumi_id = re.search(r'(\d+)', self.url).group(1)
         bangumi_data = get_bangumi_info(bangumi_id)

From 35905c2bac6021a290f1cd4893b8a0c8a17fff47 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 6 Aug 2017 00:11:55 +0800
Subject: [PATCH 0434/1225] [qq]fix bad json with fragment_count equals 0

---
 src/you_get/extractors/qq.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index f103ed0048..d62362be7d 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -18,7 +18,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     title = video_json['vl']['vi'][0]['ti']
     host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
     streams = video_json['fl']['fi']
-    seg_cnt = len(video_json['vl']['vi'][0]['cl']['ci'])
+    seg_cnt = video_json['vl']['vi'][0]['cl']['fc']
+    if seg_cnt == 0:
+        seg_cnt = 1
 
     best_quality = streams[-1]['name']
     part_format_id = streams[-1]['id']

From 38e3b4618f27cce834dc4225907854a53d391d80 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 6 Aug 2017 02:08:20 +0800
Subject: [PATCH 0435/1225] [panda]quit if cannot found room id from url

---
 src/you_get/extractors/panda.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
index 45249bd280..ba6f2b4e1c 100644
--- a/src/you_get/extractors/panda.py
+++ b/src/you_get/extractors/panda.py
@@ -3,11 +3,15 @@
 __all__ = ['panda_download']
 
 from ..common import *
+from ..util.log import *
 import json
 import time
 
 def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    roomid = url[url.rfind('/')+1:]
+    roomid = re.search('/(\d+)', url)
+    if roomid is None:
+        log.wtf('Cannot found room id for this url')
+    roomid = roomid.group(1)
     json_request_url ="http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}".format(roomid, int(time.time()))
     content = get_html(json_request_url)
     api_json = json.loads(content)

From 6d60805079042563363271ba71b245901379ce87 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 6 Aug 2017 02:33:12 +0800
Subject: [PATCH 0436/1225] [cntv]fix #2233

---
 src/you_get/extractors/cntv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index 87f1984f9d..bf926d2694 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -39,7 +39,11 @@ def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwar
          re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \
          re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url): 
-        id = r1(r'videoCenterId","(\w+)"', get_html(url))
+        page = get_content(url)
+        id = r1(r'videoCenterId","(\w+)"', page)
+        if id is None:
+            guid = re.search(r'guid\s*=\s*"([0-9a-z]+)"', page).group(1)
+            id = guid
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
         id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)
     else:

From 2344e7eaf441687d8cd817769f90b433d3bd11b7 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 6 Aug 2017 02:39:53 +0800
Subject: [PATCH 0437/1225] [mgtv]new pattern; fix #2232

---
 src/you_get/extractors/mgtv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index b0df6b28bc..8337b88747 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -27,7 +27,7 @@ class MGTV(VideoExtractor):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        vid = match1(url, 'http://www.mgtv.com/b/\d+/(\d+).html')
+        vid = match1(url, 'http://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
         if not vid:
             vid = match1(url, 'http://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
         return vid

From 450eb5671819b49092e426c38d1cd85238374952 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 6 Aug 2017 19:59:00 +0800
Subject: [PATCH 0438/1225] [qie]support match page

---
 src/you_get/extractors/qie.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/qie.py b/src/you_get/extractors/qie.py
index 2288106a09..38f703ed27 100644
--- a/src/you_get/extractors/qie.py
+++ b/src/you_get/extractors/qie.py
@@ -3,6 +3,7 @@
 
 from ..common import *
 from ..extractor import VideoExtractor
+from ..util.log import *
 
 from json import loads
 
@@ -19,13 +20,32 @@ class QiE(VideoExtractor):
     id_dic = {i['video_profile']:(i['id']) for i in stream_types}
     
     api_endpoint = 'http://www.qie.tv/api/v1/room/{room_id}'
+    game_ep = 'http://live.qq.com/game/game_details/get_game_details_info/'
 
-    @staticmethod
-    def get_vid_from_url(url):
+    def get_room_id_from_url(self, match_id):
+        meta = json.loads(get_content(self.game_ep + str(match_id)))
+        if meta['error'] != 0:
+            log.wtf('Error happens when accessing game_details api')
+        rooms = meta['data']['anchor_data']
+        for room in rooms:
+            if room['is_use_room']:
+                return room['room_id']
+        log.wtf('No room available for match {}'.format(match_id))
+
+    def get_vid_from_url(self, url):
         """Extracts video ID from live.qq.com.
         """
+        hit = re.search(r'live.qq.com/(\d+)', url)
+        if hit is not None:
+            return hit.group(1)
+        hit = re.search(r'live.qq.com/directory/match/(\d+)', url)
+        if hit is not None:
+            return self.get_room_id_from_url(hit.group(1))
         html = get_content(url)
-        return match1(html, r'room_id\":(\d+)')
+        room_id = match1(html, r'room_id\":(\d+)')
+        if room_id is None:
+            log.wtf('Unknown page {}'.format(url))
+        return room_id
 
     def download_playlist_by_url(self, url, **kwargs):
         pass
@@ -75,4 +95,4 @@ def extract(self, **kwargs):
 
 site = QiE()
 download = site.download_by_url
-download_playlist = playlist_not_supported('QiE')
\ No newline at end of file
+download_playlist = playlist_not_supported('QiE')

From d2558cc66ef671a88a6dae115dae41f3816b60e1 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 03:37:37 +0800
Subject: [PATCH 0439/1225] [extractor]set ext to mp4 if container is m3u8

---
 src/you_get/extractor.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 8c7dcf7f70..c99f46f81a 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -210,6 +210,9 @@ def download(self, **kwargs):
                 ext = self.dash_streams[stream_id]['container']
                 total_size = self.dash_streams[stream_id]['size']
 
+            if ext == 'm3u8':
+                ext = 'mp4'
+
             if not urls:
                 log.wtf('[Failed] Cannot extract video source.')
             # For legacy main()

From 828abd71d875ef52857c3756549e7795c64553b2 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 03:38:34 +0800
Subject: [PATCH 0440/1225] [qq qie_video]support QiE video

---
 src/you_get/extractors/qie_video.py | 76 +++++++++++++++++++++++++++++
 src/you_get/extractors/qq.py        |  6 ++-
 2 files changed, 81 insertions(+), 1 deletion(-)
 create mode 100644 src/you_get/extractors/qie_video.py

diff --git a/src/you_get/extractors/qie_video.py b/src/you_get/extractors/qie_video.py
new file mode 100644
index 0000000000..c5d96e708b
--- /dev/null
+++ b/src/you_get/extractors/qie_video.py
@@ -0,0 +1,76 @@
+from ..common import *
+from ..extractor import VideoExtractor
+from ..util.log import *
+
+import json
+import math
+
+class QieVideo(VideoExtractor):
+    name = 'QiE Video'
+    vid_patt = r'"stream_name":"(\d+)"'
+    title_patt = r'"title":"([^\"]+)"'
+    cdn = 'http://qietv-play.wcs.8686c.com/'
+    ep = 'http://api.qiecdn.com/api/v1/video/stream/{}'
+    stream_types = [
+        {'id':'720p', 'video_profile':'1280x720', 'container':'m3u8'},
+        {'id':'480p', 'video_profile':'853x480', 'container':'m3u8'}
+    ]
+
+    def get_vid_from_url(self):
+        hit = re.search(self.__class__.vid_patt, self.page)
+        if hit is None:
+            log.wtf('Cannot get stream_id')
+        return hit.group(1)
+
+    def get_title(self):
+        hit = re.search(self.__class__.title_patt, self.page)
+        if hit is None:
+            return self.vid
+        return hit.group(1).strip()
+
+    def prepare(self, **kwargs):
+        self.page = get_content(self.url)
+        if self.vid is None:
+            self.vid = self.get_vid_from_url()
+        self.title = self.get_title()
+        meta = json.loads(get_content(self.__class__.ep.format(self.vid)))
+        if meta['code'] != 200:
+            log.wtf(meta['message'])
+        for video in meta['result']['videos']:
+            height = video['height']
+            url = self.__class__.cdn + video['key']
+            stream_meta = dict(m3u8_url=url, size=0, container='m3u8')
+            video_profile = '{}x{}'.format(video['width'], video['height'])
+            stream_meta['video_profile'] = video_profile
+            for stream_type in self.__class__.stream_types:
+                if height // 10 == int(stream_type['id'][:-1]) // 10:
+# width 481, 482... 489 are all 480p here
+                    stream_id = stream_type['id']
+                    self.streams[stream_id] = stream_meta
+
+    def extract(self, **kwargs):
+        for stream_id in self.streams:
+            self.streams[stream_id]['src'], dur = general_m3u8_extractor(self.streams[stream_id]['m3u8_url'])
+            self.streams[stream_id]['video_profile'] += ', Duration: {}s'.format(math.floor(dur))
+
+def general_m3u8_extractor(url):
+    dur = 0
+    base_url = url[:url.rfind('/')]
+    m3u8_content = get_content(url).split('\n')
+    result = []
+    for line in m3u8_content:
+        trimmed = line.strip()
+        if len(trimmed) > 0:
+            if trimmed.startswith('#'):
+                if trimmed.startswith('#EXTINF'):
+                    t_str = re.search(r'(\d+\.\d+)', trimmed).group(1)
+                    dur += float(t_str)
+            else:
+                if trimmed.startswith('http'):
+                    result.append(trimmed)
+                else:
+                    result.append(base_url + '/' + trimmed)
+    return result, dur 
+    
+site = QieVideo()
+download_by_url = site.download_by_url
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index d62362be7d..fedaf5f9fe 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -5,6 +5,7 @@
 from ..common import *
 from ..util.log import *
 from .qie import download as qieDownload
+from .qie_video import download_by_url as qie_video_download
 from urllib.parse import urlparse,parse_qs
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
@@ -92,7 +93,10 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         return
 
     if 'live.qq.com' in url:
-        qieDownload(url, output_dir=output_dir, merge=merge, info_only=info_only)
+        if 'live.qq.com/video/v' in url:
+            qie_video_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        else:
+            qieDownload(url, output_dir=output_dir, merge=merge, info_only=info_only)
         return
 
     if 'mp.weixin.qq.com/s?' in url:

From 6f97cda395dd35a5655fa2fe922c2d52beb3d36a Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 04:15:07 +0800
Subject: [PATCH 0441/1225] [douyutv]support v.douyu.com/show/

---
 src/you_get/extractors/douyutv.py | 32 +++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 76055834f4..ae719e0d40 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -3,11 +3,43 @@
 __all__ = ['douyutv_download']
 
 from ..common import *
+from ..util.log import *
 import json
 import hashlib
 import time
+import re
+
+def douyutv_video_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    ep = 'http://vmobile.douyu.com/video/getInfo?vid='
+    patt = r'show/([0-9A-Za-z]+)'
+    title_patt = r'<h1>(.+?)</h1>'
+
+    hit = re.search(patt, url)
+    if hit is None:
+        log.wtf('Unknown url pattern')
+    vid = hit.group(1)
+
+    page = get_content(url)
+    hit = re.search(title_patt, page)
+    if hit is None:
+        title = vid
+    else:
+        title = hit.group(1)
+
+    meta = json.loads(get_content(ep + vid))
+    if meta['error'] != 0:
+        log.wtf('Error from API server')
+    m3u8_url = meta['data']['video_url']
+    print_info('Douyu Video', title, 'm3u8', 0, m3u8_url=m3u8_url)
+    if not info_only:
+        urls = general_m3u8_extractor(m3u8_url)
+        download_urls(urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs)
 
 def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    if 'v.douyu.com/show/' in url:
+        douyutv_video_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
+
     html = get_content(url)
     room_id_patt = r'"room_id"\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)

From a020c0fe32be6940a6602b0d631b798af294c62c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 13:46:15 +0800
Subject: [PATCH 0442/1225] [sina]rewrite; support sina.com.cn/zxt

---
 src/you_get/extractors/sina.py | 90 ++++++++++++++++++++++++----------
 1 file changed, 63 insertions(+), 27 deletions(-)

diff --git a/src/you_get/extractors/sina.py b/src/you_get/extractors/sina.py
index 121c5e13c1..bb94d8e386 100644
--- a/src/you_get/extractors/sina.py
+++ b/src/you_get/extractors/sina.py
@@ -3,45 +3,50 @@
 __all__ = ['sina_download', 'sina_download_by_vid', 'sina_download_by_vkey']
 
 from ..common import *
+from ..util.log import *
 
 from hashlib import md5
 from random import randint
 from time import time
+from xml.dom.minidom import parseString
+import urllib.parse
 
-def get_k(vid, rand):
-    t = str(int('{0:b}'.format(int(time()))[:-6], 2))
-    return md5((vid + 'Z6prk18aWxP278cVAH' + t + rand).encode('utf-8')).hexdigest()[:16] + t
-
-def video_info_xml(vid):
+def api_req(vid):
     rand = "0.{0}{1}".format(randint(10000, 10000000), randint(10000, 10000000))
-    url = 'http://ask.ivideo.sina.com.cn/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, get_k(vid, rand))
-    xml = get_content(url, headers=fake_headers, decoded=True)
+    t = str(int('{0:b}'.format(int(time()))[:-6], 2))
+    k = md5((vid + 'Z6prk18aWxP278cVAH' + t + rand).encode('utf-8')).hexdigest()[:16] + t
+    url = 'http://ask.ivideo.sina.com.cn/v_play.php?vid={0}&ran={1}&p=i&k={2}'.format(vid, rand, k)
+    xml = get_content(url, headers=fake_headers)
     return xml
 
 def video_info(xml):
-    urls = re.findall(r'<url>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</url>', xml)
-    name = match1(xml, r'<vname>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vname>')
-    vstr = match1(xml, r'<vstr>(?:<!\[CDATA\[)?(.+?)(?:\]\]>)?</vstr>')
-    return urls, name, vstr
+    video = parseString(xml).getElementsByTagName('video')[0]
+    result = video.getElementsByTagName('result')[0]
+    if result.firstChild.nodeValue == 'error':
+        message = video.getElementsByTagName('message')[0]
+        return None, message.firstChild.nodeValue, None
+    vname = video.getElementsByTagName('vname')[0].firstChild.nodeValue
+    durls = video.getElementsByTagName('durl')
+
+    urls = []
+    size = 0
+    for durl in durls:
+        url = durl.getElementsByTagName('url')[0].firstChild.nodeValue
+        seg_size = durl.getElementsByTagName('filesize')[0].firstChild.nodeValue
+        urls.append(url)
+        size += int(seg_size)
+
+    return urls, vname, size
 
 def sina_download_by_vid(vid, title=None, output_dir='.', merge=True, info_only=False):
     """Downloads a Sina video by its unique vid.
     http://video.sina.com.cn/
     """
-
-    xml = video_info_xml(vid)
-    sina_download_by_xml(xml, title, output_dir, merge, info_only)
-
-
-def sina_download_by_xml(xml, title, output_dir, merge, info_only):
-    urls, name, vstr = video_info(xml)
-    title = title or name
-    assert title
-    size = 0
-    for url in urls:
-        _, _, temp = url_info(url)
-        size += temp
-
+    xml = api_req(vid)
+    urls, name, size = video_info(xml)
+    if urls is None:
+        log.wtf(name)
+    title = name
     print_info(site_info, title, 'flv', size)
     if not info_only:
         download_urls(urls, title, 'flv', size, output_dir = output_dir, merge = merge)
@@ -58,9 +63,40 @@ def sina_download_by_vkey(vkey, title=None, output_dir='.', merge=True, info_onl
     if not info_only:
         download_urls([url], title, 'flv', size, output_dir = output_dir, merge = merge)
 
+def sina_zxt(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    ep = 'http://s.video.sina.com.cn/video/play?video_id='
+    frag = urllib.parse.urlparse(url).fragment
+    if not frag:
+        log.wtf('No video specified with fragment')
+    meta = json.loads(get_content(ep + frag))
+    if meta['code'] != 1:
+# Yes they use 1 for success.
+        log.wtf(meta['message'])
+    title = meta['data']['title']
+    videos = sorted(meta['data']['videos'], key = lambda i: int(i['size']))
+
+    if len(videos) == 0:
+        log.wtf('No video file returned by API server')
+
+    vid = videos[-1]['file_id']
+    container = videos[-1]['type']
+    size = int(videos[-1]['size'])
+
+    if container == 'hlv':
+        container = 'flv'
+
+    urls, _, _ = video_info(api_req(vid))
+    print_info(site_info, title, container, size)
+    if not info_only:
+        download_urls(urls, title, container, size, output_dir=output_dir, merge=merge, **kwargs)
+    return
+
 def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """Downloads Sina videos by URL.
     """
+    if 'news.sina.com.cn/zxt' in url:
+        sina_zxt(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
 
     vid = match1(url, r'vid=(\d+)')
     if vid is None:
@@ -73,8 +109,8 @@ def sina_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if vid is None:
         vid = match1(video_page, r'vid:"?(\d+)"?')
     if vid:
-        title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
-        sina_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        #title = match1(video_page, r'title\s*:\s*\'([^\']+)\'')
+        sina_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
     else:
         vkey = match1(video_page, r'vkey\s*:\s*"([^"]+)"')
         if vkey is None:

From 930c6591414a0a97d1278cd7006ce2af4a24bcc2 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 15:00:51 +0800
Subject: [PATCH 0443/1225] [xiami]remove deprecated code; fix #1165

---
 src/you_get/extractors/xiami.py | 65 ++++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
index ff9656022d..2d362b2703 100644
--- a/src/you_get/extractors/xiami.py
+++ b/src/you_get/extractors/xiami.py
@@ -28,23 +28,24 @@ def location_dec(str):
     return parse.unquote(out).replace("^", "0")
 
 def xiami_download_lyric(lrc_url, file_name, output_dir):
-    lrc = get_html(lrc_url, faker = True)
+    lrc = get_content(lrc_url, headers=fake_headers)
     filename = get_filename(file_name)
     if len(lrc) > 0:
         with open(output_dir + "/" + filename + '.lrc', 'w', encoding='utf-8') as x:
             x.write(lrc)
 
 def xiami_download_pic(pic_url, file_name, output_dir):
+    from ..util.strings import get_filename
     pic_url = pic_url.replace('_1', '')
     pos = pic_url.rfind('.')
     ext = pic_url[pos:]
-    pic = get_response(pic_url, faker = True).data
+    pic = get_content(pic_url, headers=fake_headers, decoded=False)
     if len(pic) > 0:
         with open(output_dir + "/" + file_name.replace('/', '-') + ext, 'wb') as x:
             x.write(pic)
 
-def xiami_download_song(sid, output_dir = '.', merge = True, info_only = False):
-    xml = get_html('http://www.xiami.com/song/playlist/id/%s/object_name/default/object_id/0' % sid, faker = True)
+def xiami_download_song(sid, output_dir = '.', info_only = False):
+    xml = get_content('http://www.xiami.com/song/playlist/id/%s/object_name/default/object_id/0' % sid, headers=fake_headers)
     doc = parseString(xml)
     i = doc.getElementsByTagName("track")[0]
     artist = i.getElementsByTagName("artist")[0].firstChild.nodeValue
@@ -55,24 +56,24 @@ def xiami_download_song(sid, output_dir = '.', merge = True, info_only = False):
         lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
     except:
         pass
-    type, ext, size = url_info(url, faker = True)
+    type_, ext, size = url_info(url, headers=fake_headers)
     if not ext:
         ext = 'mp3'
 
     print_info(site_info, song_title, ext, size)
     if not info_only:
         file_name = "%s - %s - %s" % (song_title, artist, album_name)
-        download_urls([url], file_name, ext, size, output_dir, merge = merge, faker = True)
+        download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
         try:
             xiami_download_lyric(lrc_url, file_name, output_dir)
         except:
             pass
 
-def xiami_download_showcollect(cid, output_dir = '.', merge = True, info_only = False):
-    html = get_html('http://www.xiami.com/song/showcollect/id/' + cid, faker = True)
+def xiami_download_showcollect(cid, output_dir = '.', info_only = False):
+    html = get_content('http://www.xiami.com/song/showcollect/id/' + cid, headers=fake_headers)
     collect_name = r1(r'<title>(.*)</title>', html)
 
-    xml = get_html('http://www.xiami.com/song/playlist/id/%s/type/3' % cid, faker = True)
+    xml = get_content('http://www.xiami.com/song/playlist/id/%s/type/3' % cid, headers=fake_headers)
     doc = parseString(xml)
     output_dir =  output_dir + "/" + "[" + collect_name + "]"
     tracks = doc.getElementsByTagName("track")
@@ -92,14 +93,14 @@ def xiami_download_showcollect(cid, output_dir = '.', merge = True, info_only =
             lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
         except:
             pass
-        type, ext, size = url_info(url, faker = True)
+        type_, ext, size = url_info(url, headers=fake_headers)
         if not ext:
             ext = 'mp3'
 
-        print_info(site_info, song_title, type, size)
+        print_info(site_info, song_title, ext, size)
         if not info_only:
             file_name = "%02d.%s - %s - %s" % (track_nr, song_title, artist, album_name)
-            download_urls([url], file_name, ext, size, output_dir, merge = merge, faker = True)
+            download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
             try:
                 xiami_download_lyric(lrc_url, file_name, output_dir)
             except:
@@ -107,17 +108,22 @@ def xiami_download_showcollect(cid, output_dir = '.', merge = True, info_only =
 
         track_nr += 1
 
-def xiami_download_album(aid, output_dir = '.', merge = True, info_only = False):
-    xml = get_html('http://www.xiami.com/song/playlist/id/%s/type/1' % aid, faker = True)
+def xiami_download_album(aid, output_dir='.', info_only=False):
+    xml = get_content('http://www.xiami.com/song/playlist/id/%s/type/1' % aid, headers=fake_headers)
     album_name = r1(r'<album_name><!\[CDATA\[(.*)\]\]>', xml)
     artist = r1(r'<artist><!\[CDATA\[(.*)\]\]>', xml)
     doc = parseString(xml)
     output_dir = output_dir + "/%s - %s" % (artist, album_name)
-    tracks = doc.getElementsByTagName("track")
+    track_list = doc.getElementsByTagName('trackList')[0]
+    tracks = track_list.getElementsByTagName("track")
     track_nr = 1
     pic_exist = False
     for i in tracks:
-        song_title = i.getElementsByTagName("title")[0].firstChild.nodeValue
+#in this xml track tag is used for both "track in a trackList" and track no
+#dirty here
+        if i.firstChild.nodeValue is not None:
+            continue
+        song_title = i.getElementsByTagName("songName")[0].firstChild.nodeValue
         url = location_dec(i.getElementsByTagName("location")[0].firstChild.nodeValue)
         try:
             lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
@@ -125,14 +131,14 @@ def xiami_download_album(aid, output_dir = '.', merge = True, info_only = False)
             pass
         if not pic_exist:
             pic_url = i.getElementsByTagName("pic")[0].firstChild.nodeValue
-        type, ext, size = url_info(url, faker = True)
+        type_, ext, size = url_info(url, headers=fake_headers)
         if not ext:
             ext = 'mp3'
 
-        print_info(site_info, song_title, type, size)
+        print_info(site_info, song_title, ext, size)
         if not info_only:
             file_name = "%02d.%s" % (track_nr, song_title)
-            download_urls([url], file_name, ext, size, output_dir, merge = merge, faker = True)
+            download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
             try:
                 xiami_download_lyric(lrc_url, file_name, output_dir)
             except:
@@ -143,26 +149,33 @@ def xiami_download_album(aid, output_dir = '.', merge = True, info_only = False)
 
         track_nr += 1
 
-def xiami_download(url, output_dir = '.', stream_type = None, merge = True, info_only = False, **kwargs):
+def xiami_download(url, output_dir='.', info_only=False, **kwargs):
+#albums
     if re.match(r'http://www.xiami.com/album/\d+', url):
         id = r1(r'http://www.xiami.com/album/(\d+)', url)
-        xiami_download_album(id, output_dir, merge, info_only)
+        xiami_download_album(id, output_dir, info_only)
+    elif re.match(r'http://www.xiami.com/album/\w+', url):
+        page = get_content(url, headers=fake_headers)
+        album_id = re.search(r'rel="canonical"\s+href="http://www.xiami.com/album/([^"]+)"', page).group(1)
+        xiami_download_album(album_id, output_dir, info_only)
 
+#collections
     if re.match(r'http://www.xiami.com/collect/\d+', url):
         id = r1(r'http://www.xiami.com/collect/(\d+)', url)
-        xiami_download_showcollect(id, output_dir, merge, info_only)
+        xiami_download_showcollect(id, output_dir, info_only)
 
+#single track
     if re.match(r'http://www.xiami.com/song/\d+\b', url):
         id = r1(r'http://www.xiami.com/song/(\d+)', url)
-        xiami_download_song(id, output_dir, merge, info_only)
+        xiami_download_song(id, output_dir, info_only)
     elif re.match(r'http://www.xiami.com/song/\w+', url):
-        html = get_html(url, faker=True)
+        html = get_content(url, headers=fake_headers)
         id = r1(r'rel="canonical" href="http://www.xiami.com/song/([^"]+)"', html)
-        xiami_download_song(id, output_dir, merge, info_only)
+        xiami_download_song(id, output_dir, info_only)
 
     if re.match('http://www.xiami.com/song/detail/id/\d+', url):
         id = r1(r'http://www.xiami.com/song/detail/id/(\d+)', url)
-        xiami_download_song(id, output_dir, merge, info_only)
+        xiami_download_song(id, output_dir, info_only)
 
 site_info = "Xiami.com"
 download = xiami_download

From 9488ce1cff492fafe1e13aaf5ea384ea387a15ea Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 8 Aug 2017 21:35:27 +0800
Subject: [PATCH 0444/1225] [youku]fix #2243

---
 src/you_get/extractors/youku.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index eebdb46fcb..ee3eb67869 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -41,7 +41,9 @@ def youku_ups(vid, ccode='0401', password=None, referer='http://v.youku.com'):
     url += '&utid=' + fetch_cna()
     url += '&client_ts=' + str(int(time.time()))
     if password is not None: url += '&password=' + password
-    return json.loads(get_content(url, headers=dict(Referer=referer)))
+    headers = dict(Referer=referer)
+    headers['User-Agent'] = fake_headers['User-Agent']
+    return json.loads(get_content(url, headers=headers))
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
@@ -161,6 +163,7 @@ def download_playlist_by_url(self, url, **kwargs):
                 traceback.print_exception(exc_type, exc_value, exc_traceback)
 
     def prepare(self, **kwargs):
+        self.ua = fake_headers['User-Agent']
         # Hot-plug cookie handler
         ssl_context = request.HTTPSHandler(
             context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))

From f1a942c0d0644303b20da0356e4d0efe86f169b2 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 9 Aug 2017 13:15:08 +0800
Subject: [PATCH 0445/1225] [vimeo]support multi streams; fix #1798

---
 src/you_get/common.py           |  10 ++-
 src/you_get/extractors/vimeo.py | 109 +++++++++++++++++++++++++++++++-
 2 files changed, 110 insertions(+), 9 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 792b19c533..a366658598 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -162,11 +162,8 @@ def rc4(key, data):
 
     return bytes(out_list)
 
-def general_m3u8_extractor(url):
-    path_len = len(url.split('/')[-1])
-    base_url = url[:-path_len]
-
-    m3u8_list = get_content(url).split('\n')
+def general_m3u8_extractor(url, headers={}):
+    m3u8_list = get_content(url, headers=headers).split('\n')
     urls = []
     for line in m3u8_list:
         line = line.strip()
@@ -174,7 +171,8 @@ def general_m3u8_extractor(url):
             if line.startswith('http'):
                 urls.append(line)
             else:
-                urls.append(base_url + line)
+                seg_url = parse.urljoin(url, line)
+                urls.append(seg_url)
     return urls
 
 def maybe_print(*s):
diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index aa8db9444b..ece1be3a9a 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -4,8 +4,10 @@
 
 from ..common import *
 from ..util.log import *
+from ..extractor import VideoExtractor
 from json import loads
 import urllib.error
+import urllib.parse
 
 access_token = 'f6785418277b72c7c87d3132c79eec24'  #By Beining
 
@@ -33,13 +35,111 @@ def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_o
         except urllib.error.URLError as e:
             log.w('{} failed with {}'.format(id, e))
 
+class VimeoExtractor(VideoExtractor):
+    stream_types = [
+        {'id': '2160p', 'video_profile': '3840x2160'},
+        {'id': '1440p', 'video_profile': '2560x1440'},
+        {'id': '1080p', 'video_profile': '1920x1080'},
+        {'id': '720p', 'video_profile': '1280x720'},
+        {'id': '540p', 'video_profile': '960x540'},
+        {'id': '360p', 'video_profile': '640x360'}
+    ]
+    name = 'Vimeo'
+
+    def prepare(self, **kwargs):
+        headers = fake_headers.copy()
+        if 'referer' in kwargs:
+            headers['Referer'] = kwargs['referer']
+
+        try:
+            page = get_content('https://vimeo.com/{}'.format(self.vid))
+            cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});'
+            cfg = json.loads(match1(page, cfg_patt))
+            video_page = get_content(cfg['player']['config_url'], headers=headers)
+            self.title = cfg['clip']['title']
+            info = json.loads(video_page)
+        except Exception as e:
+            page = get_content('https://player.vimeo.com/video/{}'.format(self.vid))
+            self.title = r1(r'<title>([^<]+)</title>', page)
+            info = json.loads(match1(page, r'var t=(\{.+?\});'))
+
+        plain = info['request']['files']['progressive']
+        for s in plain:
+            meta = dict(src=[s['url']], container='mp4')
+            meta['video_profile'] = '{}x{}'.format(s['width'], s['height'])
+            for stream in self.__class__.stream_types:
+                if s['quality'] == stream['id']:
+                    self.streams[s['quality']] = meta
+        self.master_m3u8 = info['request']['files']['hls']['cdns']
+
+    def extract(self, **kwargs):
+        for s in self.streams:
+            self.streams[s]['size'] = urls_size(self.streams[s]['src'])
+
+        master_m3u8s = []
+        for m in self.master_m3u8:
+            master_m3u8s.append(self.master_m3u8[m]['url'])
+
+        master_content = None
+        master_url = None
+
+        for master_u in master_m3u8s:
+            try:
+                master_content = get_content(master_u).split('\n')
+            except urllib.error.URLError:
+                continue
+            else:
+                master_url = master_u
+
+        if master_content is None:
+            return
+
+        lines = []
+        for line in master_content:
+            if len(line.strip()) > 0:
+                lines.append(line.strip())
+
+        pos = 0
+        while pos < len(lines):
+            if lines[pos].startswith('#EXT-X-STREAM-INF'):
+                patt = 'RESOLUTION=(\d+)x(\d+)'
+                hit = re.search(patt, lines[pos])
+                if hit is None:
+                    continue
+                width = hit.group(1)
+                height = hit.group(2)
+
+                if height in ('2160', '1440'):
+                    m3u8_url = urllib.parse.urljoin(master_url, lines[pos+1])
+                    meta = dict(m3u8_url=m3u8_url, container='m3u8')
+                    if height == '1440':
+                        meta['video_profile'] = '2560x1440'
+                    else:
+                        meta['video_profile'] = '3840x2160'
+                    meta['size'] = 0
+                    meta['src'] = general_m3u8_extractor(m3u8_url)
+                    self.streams[height+'p'] = meta
+
+                pos += 2
+            else:
+                pos += 1
+        self.streams_sorted = []
+        for stream_type in self.stream_types:
+            if stream_type['id'] in self.streams:
+                item = [('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())
+                self.streams_sorted.append(dict(item))
+
+
+
 def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
+    '''
     try:
         # normal Vimeo video
         html = get_content('https://vimeo.com/' + id)
-        config_url = unescape_html(r1(r'data-config-url="([^"]+)"', html))
-        video_page = get_content(config_url, headers=fake_headers)
-        title = r1(r'"title":"([^"]+)"', video_page)
+        cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});'
+        cfg = json.loads(match1(html, cfg_patt))
+        video_page = get_content(cfg['player']['config_url'], headers=fake_headers)
+        title = cfg['clip']['title']
         info = loads(video_page)
     except:
         # embedded player - referer may be required
@@ -59,6 +159,9 @@ def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=F
     print_info(site_info, title, type, size)
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge=merge, faker=True)
+    '''
+    site = VimeoExtractor()
+    site.download_by_vid(id, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
 def vimeo_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if re.match(r'https?://vimeo.com/channels/\w+', url):

From 87fb35769e76d4b0bb22ab2eba30dd02eb76feab Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 10 Aug 2017 22:17:15 +0800
Subject: [PATCH 0446/1225] [icourses]rewrite

---
 src/you_get/common.py              | 148 ---------
 src/you_get/extractors/icourses.py | 483 ++++++++++++++++++++++-------
 2 files changed, 363 insertions(+), 268 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a366658598..f140132d99 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -605,76 +605,6 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
         os.remove(filepath) # on Windows rename could fail if destination filepath exists
     os.rename(temp_filepath, filepath)
 
-def url_save_chunked(url, filepath, bar, dyn_callback=None, chunk_size=0, ignore_range=False, refer=None, is_part=False, faker=False, headers={}):
-    def dyn_update_url(received):
-        if callable(dyn_callback):
-            logging.debug('Calling callback %s for new URL from %s' % (dyn_callback.__name__, received))
-            return dyn_callback(received)
-    if os.path.exists(filepath):
-        if not force:
-            if not is_part:
-                if bar:
-                    bar.done()
-                print('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
-            else:
-                if bar:
-                    bar.update_received(os.path.getsize(filepath))
-            return
-        else:
-            if not is_part:
-                if bar:
-                    bar.done()
-                print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
-    elif not os.path.exists(os.path.dirname(filepath)):
-        os.mkdir(os.path.dirname(filepath))
-
-    temp_filepath = filepath + '.download'
-    received = 0
-    if not force:
-        open_mode = 'ab'
-
-        if os.path.exists(temp_filepath):
-            received += os.path.getsize(temp_filepath)
-            if bar:
-                bar.update_received(os.path.getsize(temp_filepath))
-    else:
-        open_mode = 'wb'
-
-    if faker:
-        headers = fake_headers
-    elif headers:
-        headers = headers
-    else:
-        headers = {}
-    if received:
-        url = dyn_update_url(received)
-        if not ignore_range:
-            headers['Range'] = 'bytes=' + str(received) + '-'
-    if refer:
-        headers['Referer'] = refer
-
-    response = urlopen_with_retry(request.Request(url, headers=headers))
-
-    with open(temp_filepath, open_mode) as output:
-        this_chunk = received
-        while True:
-            buffer = response.read(1024 * 256)
-            if not buffer:
-                break
-            output.write(buffer)
-            received += len(buffer)
-            if chunk_size and (received - this_chunk) >= chunk_size:
-                url = dyn_callback(received)
-                this_chunk = received
-                response = urlopen_with_retry(request.Request(url, headers=headers))
-            if bar:
-                bar.update_received(len(buffer))
-
-    assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath))
-
-    if os.access(filepath, os.W_OK):
-        os.remove(filepath) # on Windows rename could fail if destination filepath exists
-    os.rename(temp_filepath, filepath)
 
 class SimpleProgressBar:
     term_size = term.get_terminal_size()[1]
@@ -918,84 +848,6 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
 
     print()
 
-def download_urls_chunked(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
-    assert urls
-    if dry_run:
-        print('Real URLs:\n%s\n' % urls)
-        return
-
-    if player:
-        launch_player(player, urls)
-        return
-
-    title = tr(get_filename(title))
-
-    filename = '%s.%s' % (title, ext)
-    filepath = os.path.join(output_dir, filename)
-    if total_size:
-        if not force and os.path.exists(filepath[:-3] + '.mkv'):
-            print('Skipping %s: file already exists' % filepath[:-3] + '.mkv')
-            print()
-            return
-        bar = SimpleProgressBar(total_size, len(urls))
-    else:
-        bar = PiecesProgressBar(total_size, len(urls))
-
-    if len(urls) == 1:
-        parts = []
-        url = urls[0]
-        print('Downloading %s ...' % tr(filename))
-        filepath = os.path.join(output_dir, filename)
-        parts.append(filepath)
-        url_save_chunked(url, filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
-        bar.done()
-
-        if not merge:
-            print()
-            return
-        if ext == 'ts':
-            from .processor.ffmpeg import has_ffmpeg_installed
-            if has_ffmpeg_installed():
-                from .processor.ffmpeg import ffmpeg_convert_ts_to_mkv
-                if ffmpeg_convert_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
-                    for part in parts:
-                        os.remove(part)
-                else:
-                    os.remove(os.path.join(output_dir, title + '.mkv'))
-            else:
-                print('No ffmpeg is found. Conversion aborted.')
-        else:
-            print("Can't convert %s files" % ext)
-    else:
-        parts = []
-        print('Downloading %s.%s ...' % (tr(title), ext))
-        for i, url in enumerate(urls):
-            filename = '%s[%02d].%s' % (title, i, ext)
-            filepath = os.path.join(output_dir, filename)
-            parts.append(filepath)
-            #print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
-            bar.update_piece(i + 1)
-            url_save_chunked(url, filepath, bar, refer = refer, is_part = True, faker = faker, headers = headers)
-        bar.done()
-
-        if not merge:
-            print()
-            return
-        if ext == 'ts':
-            from .processor.ffmpeg import has_ffmpeg_installed
-            if has_ffmpeg_installed():
-                from .processor.ffmpeg import ffmpeg_concat_ts_to_mkv
-                if ffmpeg_concat_ts_to_mkv(parts, os.path.join(output_dir, title + '.mkv')):
-                    for part in parts:
-                        os.remove(part)
-                else:
-                    os.remove(os.path.join(output_dir, title + '.mkv'))
-            else:
-                print('No ffmpeg is found. Merging aborted.')
-        else:
-            print("Can't merge %s files" % ext)
-
-    print()
 
 def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
     assert url
diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index cb2ff74aee..cdc21a8066 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -1,148 +1,391 @@
 #!/usr/bin/env python
 from ..common import *
-from urllib import parse
+from urllib import parse, error
 import random
 from time import sleep
-import xml.etree.ElementTree as ET
 import datetime
 import hashlib
 import base64
 import logging
-from urllib import error
 import re
+from xml.dom.minidom import parseString
 
-__all__ = ['icourses_download']
+__all__ = ['icourses_download', 'icourses_playlist_download']
 
 
-def icourses_download(url, merge=False, output_dir='.', **kwargs):
+def icourses_download(url, output_dir='.', **kwargs):
+    if 'showResDetail.action' in url:
+        hit = re.search(r'id=(\d+)&courseId=(\d+)', url)
+        url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}'.format(hit.group(1), hit.group(2))
+    if re.match(r'http://www.icourses.cn/coursestatic/course_(\d+).html', url):
+        raise Exception('You can donwload it with -l flag')
     icourses_parser = ICousesExactor(url=url)
-    real_url = icourses_parser.icourses_cn_url_parser(**kwargs)
+    icourses_parser.basic_extract()
     title = icourses_parser.title
-    if real_url is not None:
-        for tries in range(0, 5):
-            try:
-                _, type_, size = url_info(real_url, faker=True)
-                break
-            except error.HTTPError:
-                logging.warning('Failed to fetch the video file! Retrying...')
-                sleep(random.Random().randint(0, 5))  # Prevent from blockage
-                real_url = icourses_parser.icourses_cn_url_parser()
-                title = icourses_parser.title
-        print_info(site_info, title, type_, size)
-        if not kwargs['info_only']:
-            download_urls_chunked([real_url], title, 'flv',
-                          total_size=size, output_dir=output_dir, refer=url, merge=merge, faker=True, ignore_range=True, chunk_size=15000000, dyn_callback=icourses_parser.icourses_cn_url_parser)
-
-
-# Why not using VideoExtractor: This site needs specical download method
+    size = None
+    for i in range(5):
+        try:
+            # use this url only for size
+            size_url = icourses_parser.generate_url(0)
+            _, type_, size = url_info(size_url, headers=fake_headers)
+        except error.HTTPError:
+            logging.warning('Failed to fetch the video file! Retrying...')
+            sleep(random.Random().randint(2, 5))  # Prevent from blockage
+        else:
+            print_info(site_info, title, type_, size)
+            break
+
+    if size is None:
+        raise Exception("Failed")
+
+    if not kwargs['info_only']:
+        real_url = icourses_parser.update_url(0)
+        headers = fake_headers.copy()
+        headers['Referer'] = url
+        download_urls_icourses(real_url, title, 'flv',total_size=size, output_dir=output_dir, max_size=15000000, dyn_callback=icourses_parser.update_url)
+    return
+
+
+def get_course_title(url, course_type, page=None):
+    if page is None:
+        try:
+            # shard course page could be gbk but with charset="utf-8"
+            page = get_content(url, decoded=False).decode('gbk')
+        except UnicodeDecodeError:
+            page = get_content(url, decoded=False).decode('utf8')
+
+    if course_type == 'shared_old':
+        patt = r'<div\s+class="top_left_til">(.+?)<\/div>'
+    elif course_type == 'shared_new':
+        patt = r'<h1>(.+?)<\/h1>'
+    else:
+        patt = r'<div\s+class="con">(.+?)<\/div>'
+
+    return re.search(patt, page).group(1)
+
+
+def public_course_playlist(url, page=None):
+    host = 'http://www.icourses.cn/'
+    patt = r'<a href="(.+?)"\s*title="(.+?)".+?>(?:.|\n)+?</a>'
+
+    if page is None:
+        page = get_content(url)
+    playlist = re.findall(patt, page)
+    return [(host+i[0], i[1]) for i in playlist]
+
+
+def public_course_get_title(url, page=None):
+    patt = r'<div\s*class="kcslbut">.+?第(\d+)讲'
+
+    if page is None:
+        page = get_content(url)
+    seq_num = int(re.search(patt, page).group(1)) - 1
+    course_main_title = get_course_title(url, 'public', page)
+    return '{}_第{}讲_{}'.format(course_main_title, seq_num+1, public_course_playlist(url, page)[seq_num][1])
+
+
+def icourses_playlist_download(url, output_dir='.', **kwargs):
+    page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
+    resid_courseid_patt = r'changeforvideo\(\'(\d+)\',\'(\d+)\',\'(\d+)\'\)'
+    ep = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'
+    change_for_video_ip = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}'
+    video_list = []
+
+    if 'viewVCourse' in url:
+        playlist = public_course_playlist(url)
+        for video in playlist:
+            icourses_download(video[0], output_dir=output_dir, **kwargs)
+        return
+    elif 'coursestatic' in url:
+        course_page = get_content(url)
+        page_navi_vars = re.search(page_type_patt, course_page)
+
+        if page_navi_vars is None:  # type 2 shared course
+            video_list = icourses_playlist_new(url, course_page)
+        else:  # type 1 shared course
+            sec_page = get_content(ep.format(page_navi_vars.group(2), page_navi_vars.group(1)))
+            video_list = re.findall(resid_courseid_patt, sec_page)
+    elif 'viewCharacterDetail.action' in url or 'changeforVideo.action' in url:
+        page = get_content(url)
+        video_list = re.findall(resid_courseid_patt, page)
+
+    if not video_list:
+        raise Exception('Unkown url pattern')
+
+    for video in video_list:
+        video_url = change_for_video_ip.format(video[0], video[1])
+        sleep(random.Random().randint(0, 5))  # Prevent from blockage
+        icourses_download(video_url, output_dir=output_dir, **kwargs)
+
+
+def icourses_playlist_new(url, page=None):
+    # 2 helpers using same interface in the js code
+    def to_chap(course_id, chap_id, mod):
+        ep = 'http://www.icourses.cn/jpk/viewCharacterDetail2.action?courseId={}&characId={}&mod={}'
+        req = post_content(ep.format(course_id, chap_id, mod), post_data={})
+        return req
+
+    def to_sec(course_id, chap_id, mod):
+        ep = 'http://www.icourses.cn/jpk/viewCharacterDetail2.action?courseId={}&characId={}&mod={}'
+        req = post_content(ep.format(course_id, chap_id, mod), post_data={})
+        return req
+
+    def show_sec(course_id, chap_id):
+        ep = 'http://www.icourses.cn/jpk/getSectionNode.action?courseId={}&characId={}&mod=2'
+        req = post_content(ep.format(course_id, chap_id), post_data={})
+        return req
+
+    if page is None:
+        page = get_content(url)
+    chap_patt = r'<h3>.+?id="parent_row_(\d+)".+?onclick="(\w+)\((.+)\)"'
+    to_chap_patt = r'this,(\d+),(\d+),(\d)'
+    show_sec_patt = r'this,(\d+),(\d+)'
+    res_patt = r'res_showResDetail\(\'(\d+)\',\'.+?\',\'\d+\',\'mp4\',\'(\d+)\'\)'
+    l = re.findall(chap_patt, page)
+    for i in l:
+        if i[1] == 'ajaxtocharac':
+            hit = re.search(to_chap_patt, i[2])
+            page = to_chap(hit.group(1), hit.group(2), hit.group(3))
+            hit_list = re.findall(res_patt, page)
+            if hit_list:
+                return get_playlist(hit_list[0][0], hit_list[0][1])
+            for hit in hit_list:
+                print(hit)
+        elif i[1] == 'showSectionNode2':
+            hit = re.search(show_sec_patt, i[2])
+            page = show_sec(hit.group(1), hit.group(2))
+            # print(page)
+            patt = r'ajaxtosection\(this,(\d+),(\d+),(\d+)\)'
+            hit_list = re.findall(patt, page)
+            # print(hit_list)
+            for hit in hit_list:
+                page = to_sec(hit[0], hit[1], hit[2])
+                vlist = re.findall(res_patt, page)
+                if vlist:
+                    return get_playlist(vlist[0][0], vlist[0][1])
+    raise Exception("No video found in this playlist")
+
+
+def get_playlist(res_id, course_id):
+    ep = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}'
+    req = get_content(ep.format(res_id, course_id))
+
+    patt = r'<a.+?changeforvideo\(\'(\d+)\',\'(\d+)\',\'(\d+)\'\).+?title=\"(.+?)\"'
+    return re.findall(patt, req)
+
+
 class ICousesExactor(object):
+    PLAYER_BASE_VER = '150606-1'
+    ENCRYPT_MOD_VER = '151020'
+    ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
 
     def __init__(self, url):
         self.url = url
         self.title = ''
+        self.flashvars = ''
+        self.api_data = {}
+        self.media_url = ''
+        self.common_args = {}
+        self.enc_mode = True
+        self.page = get_content(self.url)
         return
 
-    def icourses_playlist_download(self, **kwargs):
-        html = get_content(self.url)
-        page_type_patt = r'showSectionNode\(this,(\d+),(\d+)\)'
-        video_js_number = r'changeforvideo\((.*?)\)'
-        fs_flag = r'<input type="hidden" value=(\w+) id="firstShowFlag">'
-        page_navi_vars = re.search(pattern=page_type_patt, string=html)
-        dummy_page = 'http://www.icourses.cn/jpk/viewCharacterDetail.action?sectionId={}&courseId={}'.format(
-            page_navi_vars.group(2), page_navi_vars.group(1))
-        html = get_content(dummy_page)
-        fs_status = match1(html, fs_flag)
-        video_list = re.findall(pattern=video_js_number, string=html)
-        for video in video_list:
-            video_args = video.replace('\'', '').split(',')
-            video_url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}&firstShowFlag={}'.format(
-                video_args[0], video_args[1], fs_status or '1')
-            sleep(random.Random().randint(0, 5))  # Prevent from blockage
-            icourses_download(video_url, **kwargs)
-
-    def icourses_cn_url_parser(self, received=0, **kwargs):
-        PLAYER_BASE_VER = '150606-1'
-        ENCRYPT_MOD_VER = '151020'
-        ENCRYPT_SALT = '3DAPmXsZ4o'  # It took really long time to find this...
-        html = get_content(self.url)
-        if re.search(pattern=r'showSectionNode\(.*\)', string=html):
-            logging.warning('Switching to playlist mode!')
-            return self.icourses_playlist_download(**kwargs)
-        flashvars_patt = r'var\ flashvars\=((.|\n)*)};'
-        server_time_patt = r'MPlayer.swf\?v\=(\d+)'
-        uuid_patt = r'uuid:(\d+)'
-        other_args_patt = r'other:"(.*)"'
-        res_url_patt = r'IService:\'([^\']+)'
+    def get_title(self):
+        if 'viewVCourse' in self.url:
+            self.title = public_course_get_title(self.url, self.page)
+            return
         title_a_patt = r'<div class="con"> <a.*?>(.*?)</a>'
         title_b_patt = r'<div class="con"> <a.*?/a>((.|\n)*?)</div>'
-        title_a = match1(html, title_a_patt).strip()
-        title_b = match1(html, title_b_patt).strip()
-        title = title_a + title_b  # WIP, FIXME
-        title = re.sub('( +|\n|\t|\r|\&nbsp\;)', '',
-                       unescape_html(title).replace(' ', ''))
-        server_time = match1(html, server_time_patt)
-        flashvars = match1(html, flashvars_patt)
-        uuid = match1(flashvars, uuid_patt)
-        other_args = match1(flashvars, other_args_patt)
-        res_url = match1(flashvars, res_url_patt)
-        url_parts = {'v': server_time, 'other': other_args,
-                     'uuid': uuid, 'IService': res_url}
-        req_url = '%s?%s' % (res_url, parse.urlencode(url_parts))
-        logging.debug('Requesting video resource location...')
-        xml_resp = get_html(req_url)
-        xml_obj = ET.fromstring(xml_resp)
-        logging.debug('The result was {}'.format(xml_obj.get('status')))
-        if xml_obj.get('status') != 'success':
-            raise ValueError('Server returned error!')
-        if received:
-            play_type = 'seek'
-        else:
-            play_type = 'play'
-            received -= 1
-        common_args = {'lv': PLAYER_BASE_VER, 'ls': play_type,
-                       'lt': datetime.datetime.now().strftime('%m-%d/%H:%M:%S'),
-                       'start': received + 1}
-        media_host = xml_obj.find(".//*[@name='host']").text
-        media_url = media_host + xml_obj.find(".//*[@name='url']").text
-        # This is what they called `SSLModule`... But obviously, just a kind of
-        # encryption, takes absolutely no effect in protecting data intergrity
-        if xml_obj.find(".//*[@name='ssl']").text != 'true':
-            logging.debug('The encryption mode is disabled')
-            # when the so-called `SSLMode` is not activated, the parameters, `h`
-            # and `p` can be found in response
-            arg_h = xml_obj.find(".//*[@name='h']").text
-            assert arg_h
-            arg_r = xml_obj.find(".//*[@name='p']").text or ENCRYPT_MOD_VER
-            url_args = common_args.copy()
-            url_args.update({'h': arg_h, 'r': arg_r})
-            final_url = '{}?{}'.format(
-                media_url, parse.urlencode(url_args))
-            self.title = title
-            return final_url
-        # when the `SSLMode` is activated, we need to receive the timestamp and the
-        # time offset (?) value from the server
-        logging.debug('The encryption mode is in effect')
-        ssl_callback = get_html(
-            '{}/ssl/ssl.shtml'.format(media_host)).split(',')
-        ssl_timestamp = int(datetime.datetime.strptime(
-            ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
-        sign_this = ENCRYPT_SALT + \
-            parse.urlparse(media_url).path + str(ssl_timestamp)
-        arg_h = base64.b64encode(hashlib.md5(
-            bytes(sign_this, 'utf-8')).digest())
-        # Post-processing, may subject to change, so leaving this alone...
-        arg_h = arg_h.decode('utf-8').strip('=').replace('+',
-                                                         '-').replace('/', '_')
-        arg_r = ssl_timestamp
-        url_args = common_args.copy()
-        url_args.update({'h': arg_h, 'r': arg_r, 'p': ENCRYPT_MOD_VER})
-        final_url = '{}?{}'.format(
-            media_url, parse.urlencode(url_args))
-        logging.debug('Crafted URL: {}'.format(final_url))
+        title_a = match1(self.page, title_a_patt).strip()
+        title_b = match1(self.page, title_b_patt).strip()
+        title = title_a + title_b
+        title = re.sub('( +|\n|\t|\r|&nbsp;)', '', unescape_html(title).replace(' ', ''))
         self.title = title
-        return final_url
 
+    def get_flashvars(self):
+        patt = r'var flashvars\s*=\s*(\{(?:.|\n)+?\});'
+        hit = re.search(patt, self.page)
+        if hit is None:
+            raise Exception('Cannot find flashvars')
+        flashvar_str = hit.group(1)
+
+        uuid = re.search(r'uuid\s*:\s*\"?(\w+)\"?', flashvar_str).group(1)
+        other = re.search(r'other\s*:\s*"(.*?)"', flashvar_str).group(1)
+        isvc = re.search(r'IService\s*:\s*\'(.+?)\'', flashvar_str).group(1)
+
+        player_time_patt = r'MPlayer.swf\?v\=(\d+)'
+        player_time = re.search(player_time_patt, self.page).group(1)
+
+        self.flashvars = dict(IService=isvc, uuid=uuid, other=other, v=player_time)
+
+    def api_req(self, url):
+        xml_str = get_content(url)
+        dom = parseString(xml_str)
+        status = dom.getElementsByTagName('result')[0].getAttribute('status')
+        if status != 'success':
+            raise Exception('API returned fail')
+
+        api_res = {}
+        meta = dom.getElementsByTagName('metadata')
+        for m in meta:
+            key = m.getAttribute('name')
+            val = m.firstChild.nodeValue
+            api_res[key] = val
+        self.api_data = api_res
+
+    def basic_extract(self):
+        self.get_title()
+        self.get_flashvars()
+        api_req_url = '{}?{}'.format(self.flashvars['IService'], parse.urlencode(self.flashvars))
+        self.api_req(api_req_url)
+
+    def do_extract(self, received=0):
+        self.basic_extract()
+        return self.generate_url(received)
+
+    def update_url(self, received):
+        args = self.common_args.copy()
+        play_type = 'empty' if received else 'play'
+        received = received if received else -1
+        args['ls'] = play_type
+        args['start'] = received + 1
+        args['lt'] = self.get_date_str()
+        if self.enc_mode:
+            ssl_ts, sign = self.get_sign(self.media_url)
+            extra_args = dict(h=sign, r=ssl_ts, p=self.__class__.ENCRYPT_MOD_VER)
+            args.update(extra_args)
+        return '{}?{}'.format(self.media_url, parse.urlencode(args))
+
+    @classmethod
+    def get_date_str(self):
+        fmt_str = '%-m-%-d/%-H:%-M:%-S'
+        return datetime.datetime.now().strftime(fmt_str)
+
+    def generate_url(self, received):
+        media_host = self.get_media_host(self.api_data['host'])
+        media_url = media_host + self.api_data['url']
+        self.media_url = media_url
+
+        common_args = dict(lv=self.__class__.PLAYER_BASE_VER)
+        h = self.api_data.get('h')
+        r = self.api_data.get('p', self.__class__.ENCRYPT_MOD_VER)
+
+        if self.api_data['ssl'] != 'true':
+            self.enc_mode = False
+            common_args.update(dict(h=h, r=r))
+        else:
+            self.enc_mode = True
+            common_args['p'] = self.__class__.ENCRYPT_MOD_VER
+        self.common_args = common_args
+        return self.update_url(received)
+
+    def get_sign(self, media_url):
+        media_host = parse.urlparse(media_url).netloc
+        ran = random.randint(0, 9999999)
+        ssl_callback = get_content('http://{}/ssl/ssl.shtml?r={}'.format(media_host, ran)).split(',')
+        ssl_ts = int(datetime.datetime.strptime(ssl_callback[1], "%b %d %H:%M:%S %Y").timestamp() + int(ssl_callback[0]))
+        sign_this = self.__class__.ENCRYPT_SALT + parse.urlparse(media_url).path + str(ssl_ts)
+        arg_h = base64.b64encode(hashlib.md5(bytes(sign_this, 'utf-8')).digest(), altchars=b'-_')
+        return ssl_ts, arg_h.decode('utf-8').strip('=')
+
+    def get_media_host(self, ori_host):
+        res = get_content(ori_host + '/ssl/host.shtml').strip()
+        path = parse.urlparse(ori_host).path
+        return ''.join([res, path])
+
+
+def download_urls_icourses(url, title, ext, total_size, output_dir='.', headers=None, **kwargs):
+    if dry_run or player:
+        log.wtf('Non standard protocol')
+
+    title = get_filename(title)
+
+    filename = '%s.%s' % (title, ext)
+    filepath = os.path.join(output_dir, filename)
+    if not force and os.path.exists(filepath):
+        print('Skipping {}: file already exists\n'.format(filepath))
+        return
+    bar = SimpleProgressBar(total_size, 1)
+    print('Downloading %s ...' % tr(filename))
+    url_save_icourses(url, filepath, bar, total_size, headers=headers, **kwargs)
+    bar.done()
+
+    print()
+
+
+def url_save_icourses(url, filepath, bar, total_size, dyn_callback=None, is_part=False, max_size=0, headers=None):
+    def dyn_update_url(received):
+        if callable(dyn_callback):
+            logging.debug('Calling callback %s for new URL from %s' % (dyn_callback.__name__, received))
+            return dyn_callback(received)
+    if bar is None:
+        bar = DummyProgressBar()
+    if os.path.exists(filepath):
+        if not force:
+            if not is_part:
+                bar.done()
+                print('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
+            else:
+                filesize = os.path.getsize(filepath)
+                bar.update_received(filesize)
+            return
+        else:
+            if not is_part:
+                bar.done()
+                print('Overwriting %s' % os.path.basename(filepath), '...')
+    elif not os.path.exists(os.path.dirname(filepath)):
+        os.mkdir(os.path.dirname(filepath))
+
+    temp_filepath = filepath + '.download'
+    received = 0
+    if not force:
+        open_mode = 'ab'
+
+        if os.path.exists(temp_filepath):
+            tempfile_size = os.path.getsize(temp_filepath)
+            received += tempfile_size
+            bar.update_received(tempfile_size)
+    else:
+        open_mode = 'wb'
+
+    if received:
+        url = dyn_update_url(received)
+
+    if headers is None:
+        headers = {}
+    response = urlopen_with_retry(request.Request(url, headers=headers))
+# Do not update content-length here.
+# Only the 1st segment's content-length is the content-length of the file.
+# For other segments, content-length is the standard one, 15 * 1024 * 1024
+
+    with open(temp_filepath, open_mode) as output:
+        before_this_uri = received
+# received - before_this_uri is size of the buf we get from one uri
+        while True:
+            update_bs = 256 * 1024
+            left_bytes = total_size - received
+            to_read = left_bytes if left_bytes <= update_bs else update_bs
+# calc the block size to read -- The server can fail to send an EOF
+            buffer = response.read(to_read)
+            if not buffer:
+                logging.debug('Got EOF from server')
+                break
+            output.write(buffer)
+            received += len(buffer)
+            bar.update_received(len(buffer))
+            if received >= total_size:
+                break
+            if max_size and (received - before_this_uri) >= max_size:
+                url = dyn_update_url(received)
+                before_this_uri = received
+                response = urlopen_with_retry(request.Request(url, headers=headers))
+
+    assert received == os.path.getsize(temp_filepath), '%s == %s' % (received, os.path.getsize(temp_filepath))
+
+    if os.access(filepath, os.W_OK):
+        os.remove(filepath)  # on Windows rename could fail if destination filepath exists
+    os.rename(temp_filepath, filepath)
 
 site_info = 'icourses.cn'
 download = icourses_download
-# download_playlist = icourses_playlist_download
+download_playlist = icourses_playlist_download

From 8af779ecdc61320b27a40f1299b7e6093d97f731 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 10 Aug 2017 23:36:06 +0800
Subject: [PATCH 0447/1225] [icourses]compact problem about strftime

---
 src/you_get/extractors/icourses.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index cdc21a8066..c6321a6e26 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -259,7 +259,12 @@ def update_url(self, received):
     @classmethod
     def get_date_str(self):
         fmt_str = '%-m-%-d/%-H:%-M:%-S'
-        return datetime.datetime.now().strftime(fmt_str)
+        now = datetime.datetime.now()
+        try:
+            date_str =  now.strftime(fmt_str)
+        except ValueError:  # msvcrt
+            date_str = '{}-{}/{}:{}:{}'.format(now.month, now.day, now.hour, now.minute, now.second)
+        return date_str
 
     def generate_url(self, received):
         media_host = self.get_media_host(self.api_data['host'])

From 00d62706ca9cd346dac74a1e1349a3296f2e4645 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 11 Aug 2017 00:05:15 +0800
Subject: [PATCH 0448/1225] [common]fix #954

---
 src/you_get/common.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f140132d99..b8ee9f35f7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1046,12 +1046,9 @@ def print_more_compatible(*args, **kwargs):
     return ret
 
 
-
 def download_main(download, download_playlist, urls, playlist, **kwargs):
     for url in urls:
-        if url.startswith('https://'):
-            url = url[8:]
-        if not url.startswith('http://'):
+        if re.match(r'https?://', url) is None:
             url = 'http://' + url
 
         if playlist:
@@ -1315,7 +1312,7 @@ def url_to_module(url):
         video_host = r1(r'https?://([^/]+)/', url)
         video_url = r1(r'https?://[^/]+(.*)', url)
         assert video_host and video_url
-    except:
+    except AssertionError:
         url = google_search(url)
         video_host = r1(r'https?://([^/]+)/', url)
         video_url = r1(r'https?://[^/]+(.*)', url)
@@ -1330,7 +1327,10 @@ def url_to_module(url):
         return import_module('.'.join(['you_get', 'extractors', SITES[k]])), url
     else:
         import http.client
-        conn = http.client.HTTPConnection(video_host)
+        if url.startswith('https://'):
+            conn = http.client.HTTPSConnection(video_host)
+        else:
+            conn = http.client.HTTPConnection(video_host)
         conn.request("HEAD", video_url, headers=fake_headers)
         res = conn.getresponse()
         location = res.getheader('location')

From d41feb76841871bc1c06645cd6249d338b083abb Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 11 Aug 2017 00:29:31 +0800
Subject: [PATCH 0449/1225] [common]fix #1603

---
 src/you_get/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index b8ee9f35f7..e8d294dc79 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1327,6 +1327,7 @@ def url_to_module(url):
         return import_module('.'.join(['you_get', 'extractors', SITES[k]])), url
     else:
         import http.client
+        video_host = r1(r'https?://([^/]+)/', url) # .cn could be removed
         if url.startswith('https://'):
             conn = http.client.HTTPSConnection(video_host)
         else:

From 94739f5b462a22f54bee396428ec19ee1f1b4ae2 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 11 Aug 2017 04:43:54 +0800
Subject: [PATCH 0450/1225] [acfun]remove useless import

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index e4f0b0d77a..7ae541d6b9 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -8,7 +8,7 @@
 from .qq import qq_download_by_vid
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_iid
-from .youku import youku_download_by_vid, youku_open_download_by_vid
+from .youku import youku_download_by_vid
 
 import json
 import re

From 10a32e9de358b975695700efd9b8373744d99449 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 11 Aug 2017 13:45:37 +0800
Subject: [PATCH 0451/1225] [youku]rewrite youku

---
 src/you_get/extractors/youku.py | 473 ++++++++++++--------------------
 1 file changed, 173 insertions(+), 300 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index ee3eb67869..95cab6b2ee 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -4,20 +4,20 @@
 from ..common import *
 from ..extractor import VideoExtractor
 
-import base64
-import ssl
 import time
 import traceback
 import json
 import urllib.request
 import urllib.parse
 
-def quote_cna(cna):
-    if '%' in cna:
-        return cna
-    return urllib.parse.quote(cna)
 
 def fetch_cna():
+
+    def quote_cna(val):
+        if '%' in val:
+            return val
+        return urllib.parse.quote(val)
+
     if cookies:
         for cookie in cookies:
             if cookie.name == 'cna' and cookie.domain == '.youku.com':
@@ -35,186 +35,131 @@ def fetch_cna():
     log.w('It seems that the client failed to fetch a cna cookie. Please load your own cookie if possible')
     return quote_cna('DOG4EdW4qzsCAbZyXbU+t7Jt')
 
-def youku_ups(vid, ccode='0401', password=None, referer='http://v.youku.com'):
-    url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(vid, ccode)
-    url += '&client_ip=192.168.1.1'
-    url += '&utid=' + fetch_cna()
-    url += '&client_ts=' + str(int(time.time()))
-    if password is not None: url += '&password=' + password
-    headers = dict(Referer=referer)
-    headers['User-Agent'] = fake_headers['User-Agent']
-    return json.loads(get_content(url, headers=headers))
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
+    mobile_ua = 'Mozilla/5.0 (iPad; CPU OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Mobile/14B100'
 
     # Last updated: 2015-11-24
     stream_types = [
-        {'id': 'mp4hd3', 'alias-of' : 'hd3'},
+        {'id': 'mp4hd3', 'alias-of': 'hd3'},
         {'id': 'hd3',    'container': 'flv', 'video_profile': '1080P'},
-        {'id': 'mp4hd2', 'alias-of' : 'hd2'},
+        {'id': 'mp4hd2', 'alias-of': 'hd2'},
         {'id': 'hd2',    'container': 'flv', 'video_profile': '超清'},
-        {'id': 'mp4hd',  'alias-of' : 'mp4'},
+        {'id': 'mp4hd',  'alias-of': 'mp4'},
         {'id': 'mp4',    'container': 'mp4', 'video_profile': '高清'},
         {'id': 'flvhd',  'container': 'flv', 'video_profile': '标清'},
         {'id': 'flv',    'container': 'flv', 'video_profile': '标清'},
-        {'id': '3gphd',  'container': '3gp', 'video_profile': '标清（3GP）'},
+        {'id': '3gphd',  'container': 'mp4', 'video_profile': '标清（3GP）'},
     ]
 
-    f_code_1 = 'becaf9be'
-    f_code_2 = 'bf7e5f01'
-
-    ctype = 12  #differ from 86
-
-    def trans_e(a, c):
-        """str, str->str
-        This is an RC4 encryption."""
-        f = h = 0
-        b = list(range(256))
-        result = ''
-        while h < 256:
-            f = (f + b[h] + ord(a[h % len(a)])) % 256
-            b[h], b[f] = b[f], b[h]
-            h += 1
-        q = f = h = 0
-        while q < len(c):
-            h = (h + 1) % 256
-            f = (f + b[h]) % 256
-            b[h], b[f] = b[f], b[h]
-            if isinstance(c[q], int):
-                result += chr(c[q] ^ b[(b[h] + b[f]) % 256])
-            else:
-                result += chr(ord(c[q]) ^ b[(b[h] + b[f]) % 256])
-            q += 1
-
-        return result
-
-    def generate_ep(self, fileid, sid, token):
-        ep = parse.quote(base64.b64encode(
-            ''.join(self.__class__.trans_e(
-                self.f_code_2,  #use the 86 fcode if using 86
-                sid + '_' + fileid + '_' + token)).encode('latin1')),
-            safe='~()*!.\''
-        )
-        return ep
-
-    # Obsolete -- used to parse m3u8 on pl.youku.com
-    def parse_m3u8(m3u8):
-        return re.findall(r'(http://[^?]+)\?ts_start=0', m3u8)
-
-    def oset(xs):
-        """Turns a list into an ordered set. (removes duplicates)"""
-        mem = set()
-        for x in xs:
-            if x not in mem:
-                mem.add(x)
-        return mem
-
-    def get_vid_from_url(url):
-        """Extracts video ID from URL.
-        """
-        return match1(url, r'youku\.com/v_show/id_([a-zA-Z0-9=]+)') or \
-          match1(url, r'player\.youku\.com/player\.php/sid/([a-zA-Z0-9=]+)/v\.swf') or \
-          match1(url, r'loader\.swf\?VideoIDS=([a-zA-Z0-9=]+)') or \
-          match1(url, r'player\.youku\.com/embed/([a-zA-Z0-9=]+)')
-
-    def get_playlist_id_from_url(url):
-        """Extracts playlist ID from URL.
-        """
-        return match1(url, r'youku\.com/albumlist/show\?id=([a-zA-Z0-9=]+)')
-
-    def download_playlist_by_url(self, url, **kwargs):
-        self.url = url
-
-        try:
-            playlist_id = self.__class__.get_playlist_id_from_url(self.url)
-            assert playlist_id
-            video_page = get_content('http://list.youku.com/albumlist/show?id=%s' % playlist_id)
-            videos = Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', video_page))
-            # Parse multi-page playlists
-            last_page_url = re.findall(r'href="(/albumlist/show\?id=%s[^"]+)" title="末页"' % playlist_id, video_page)[0]
-            num_pages = int(re.findall(r'page=([0-9]+)\.htm', last_page_url)[0])
-            if (num_pages > 0):
-                # download one by one
-                for pn in range(2, num_pages + 1):
-                    extra_page_url = re.sub(r'page=([0-9]+)\.htm', r'page=%s.htm' % pn, last_page_url)
-                    extra_page = get_content('http://list.youku.com' + extra_page_url)
-                    videos |= Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', extra_page))
-        except:
-            # Show full list of episodes
-            if match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)'):
-                ep_id = match1(url, r'youku\.com/show_page/id_([a-zA-Z0-9=]+)')
-                url = 'http://www.youku.com/show_episode/id_%s' % ep_id
-
-            video_page = get_content(url)
-            videos = Youku.oset(re.findall(r'href="(http://v\.youku\.com/[^?"]+)', video_page))
-
-        self.title = r1(r'<meta name="title" content="([^"]+)"', video_page) or \
-                     r1(r'<title>([^<]+)', video_page)
-        self.p_playlist()
-        for video in videos:
-            index = parse_query_param(video, 'f')
-            try:
-                self.__class__().download_by_url(video, index=index, **kwargs)
-            except KeyboardInterrupt:
-                raise
-            except:
-                exc_type, exc_value, exc_traceback = sys.exc_info()
-                traceback.print_exception(exc_type, exc_value, exc_traceback)
+    def __init__(self):
+        super().__init__()
+
+        self.ua = self.__class__.mobile_ua
+        self.referer = 'http://v.youku.com'
+
+        self.page = None
+        self.video_list = None
+        self.password = None
+        self.api_data = None
+        self.api_error_code = None
+        self.api_error_msg = None
+
+        self.ccode = '0401'
+        self.utid = None
+
+    def youku_ups(self):
+        url = 'https://ups.youku.com/ups/get.json?vid={}&ccode={}'.format(self.vid, self.ccode)
+        url += '&client_ip=192.168.1.1'
+        url += '&utid=' + self.utid
+        url += '&client_ts=' + str(int(time.time()))
+        if self.password_protected:
+            url += '&password=' + self.password
+        headers = dict(Referer=self.referer)
+        headers['User-Agent'] = self.ua
+        api_meta = json.loads(get_content(url, headers=headers))
+
+        self.api_data = api_meta['data']
+        data_error = self.api_data.get('error')
+        if data_error:
+            self.api_error_code = data_error.get('code')
+            self.api_error_msg = data_error.get('note')
+        if 'videos' in self.api_data:
+            if 'list' in self.api_data['videos']:
+                self.video_list = self.api_data['videos']['list']
+
+    def get_vid_from_url(self):
+        # It's unreliable. check #1633
+        b64p = r'([a-zA-Z0-9=]+)'
+        p_list = [r'youku\.com/v_show/id_'+b64p,
+                  r'player\.youku\.com/player\.php/sid/'+b64p+r'/v\.swf',
+                  r'loader\.swf\?VideoIDS='+b64p,
+                  r'player\.youku\.com/embed/'+b64p]
+        if not self.url:
+            raise Exception('No url')
+        for p in p_list:
+            hit = re.search(p, self.url)
+            if hit is not None:
+                self.vid = hit.group(1)
+                return
+
+    def get_vid_from_page(self):
+        if not self.url:
+            raise Exception('No url')
+        self.page = get_content(self.url)
+        hit = re.search(r'videoId2:"([A-Za-z0-9=]+)"', self.page)
+        if hit is not None:
+            self.vid = hit.group(1)
 
     def prepare(self, **kwargs):
-        self.ua = fake_headers['User-Agent']
-        # Hot-plug cookie handler
-        ssl_context = request.HTTPSHandler(
-            context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
-        cookie_handler = request.HTTPCookieProcessor()
-        if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
-            proxy = parse_host(kwargs['extractor_proxy'])
-            proxy_handler = request.ProxyHandler({
-                'http': '%s:%s' % proxy,
-                'https': '%s:%s' % proxy,
-            })
-        else:
-            proxy_handler = request.ProxyHandler({})
-        if not request._opener:
-            opener = request.build_opener(proxy_handler)
-            request.install_opener(opener)
-        for handler in (ssl_context, cookie_handler, proxy_handler):
-            request._opener.add_handler(handler)
-        request._opener.addheaders = [('Cookie','__ysuid={}'.format(time.time()))]
-
         assert self.url or self.vid
 
         if self.url and not self.vid:
-            self.vid = self.__class__.get_vid_from_url(self.url)
+            self.get_vid_from_url()
 
             if self.vid is None:
-                self.download_playlist_by_url(self.url, **kwargs)
-                exit(0)
+                self.get_vid_from_page()
+
+                if self.vid is None:
+                    log.wtf('Cannot fetch vid')
+
         if kwargs.get('src') and kwargs['src'] == 'tudou':
-            data = youku_ups(self.vid, '0402')['data']
-        else:
-            data = youku_ups(self.vid)['data']
-        if data.get('stream') is None:
-            if data.get('error'):
-                if data['error']['code'] == -2002:
-                    self.password_protected = True
-                    self.password = kwargs.get("password", None)
-                    if not self.password:
-                        self.password = input(log.sprint('Password: ', log.YELLOW))
-                    data = youku_ups(self.vid, password=self.password)['data']
-                    if data.get('error'):
-                        log.wtf(data['error']['note'])
-                else:
-                    log.wtf(data['error']['note'])
+            self.ccode = '0402'
+
+        if kwargs.get('password') and kwargs['password']:
+            self.password_protected = True
+            self.password = kwargs['password']
+
+        self.utid = fetch_cna()
+        self.youku_ups()
+
+        if self.api_data.get('stream') is None:
+            if self.api_error_code == -6001:  # wrong vid parsed from the page
+                vid_from_url = self.vid
+                self.get_vid_from_page()
+                if vid_from_url == self.vid:
+                    log.wtf(self.api_error_msg)
+                self.youku_ups()
+
+        if self.api_data.get('stream') is None:
+            if self.api_error_code == -2002:  # wrong password
+                self.password_protected = True
+                # it can be True already(from cli). offer another chance to retry
+                self.password = input(log.sprint('Password: ', log.YELLOW))
+                self.youku_ups()
+
+        if self.api_data.get('stream') is None:
+            if self.api_error_msg:
+                log.wtf(self.api_error_msg)
             else:
                 log.wtf('Unknown error')
 
-        self.title = data['video']['title']
+        self.title = self.api_data['video']['title']
         stream_types = dict([(i['id'], i) for i in self.stream_types])
-        audio_lang = data['stream'][0]['audio_lang']
+        audio_lang = self.api_data['stream'][0]['audio_lang']
 
-        for stream in data['stream']:
+        for stream in self.api_data['stream']:
             stream_id = stream['stream_type']
             is_preview = False
             if stream_id in stream_types and stream['audio_lang'] == audio_lang:
@@ -254,144 +199,72 @@ def prepare(self, **kwargs):
                 log.w('{} is a preview'.format(stream_id))
 
         # Audio languages
-        if 'dvd' in data and 'audiolang' in data['dvd']:
-            self.audiolang = data['dvd']['audiolang']
-            for i in self.audiolang:
-                i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
-    '''
-    def extract(self, **kwargs):
-        if 'stream_id' in kwargs and kwargs['stream_id']:
-            # Extract the stream
-            stream_id = kwargs['stream_id']
-
-            if stream_id not in self.streams:
-                log.e('[Error] Invalid video format.')
-                log.e('Run \'-i\' command with no specific video format to view all available formats.')
-                exit(2)
-        else:
-            # Extract stream with the best quality
-            stream_id = self.streams_sorted[0]['id']
-
-        e_code = self.__class__.trans_e(
-            self.f_code_1,
-            base64.b64decode(bytes(self.ep, 'ascii'))
-        )
-        sid, token = e_code.split('_')
-        while True:
-            try:
-                ksegs = []
-                pieces = self.streams[stream_id]['pieces']
-                for piece in pieces:
-                    segs = piece['segs']
-                    seg_count = len(segs)
-                    for no in range(0, seg_count):
-                        k = segs[no]['key']
-                        fileid = segs[no]['fileid']
-                        if k == -1:
-                            # we hit the paywall; stop here
-                            log.w('Skipping %d out of %d segments due to paywall' %
-                                  (seg_count - no, seg_count))
-                            break
-                        ep = self.__class__.generate_ep(self, fileid,
-                                                        sid, token)
-                        q = parse.urlencode(dict(
-                            ctype = self.ctype,
-                            ev    = 1,
-                            K     = k,
-                            ep    = parse.unquote(ep),
-                            oip   = str(self.ip),
-                            token = token,
-                            yxon  = 1
-                        ))
-                        u = 'http://k.youku.com/player/getFlvPath/sid/{sid}_00' \
-                            '/st/{container}/fileid/{fileid}?{q}'.format(
-                                sid       = sid,
-                                container = self.streams[stream_id]['container'],
-                                fileid    = fileid,
-                                q         = q
-                            )
-                        ksegs += [i['server'] for i in json.loads(get_content(u))]
-            except error.HTTPError as e:
-                # Use fallback stream data in case of HTTP 404
-                log.e('[Error] ' + str(e))
-                self.streams = {}
-                self.streams = self.streams_fallback
-            except KeyError:
-                # Move on to next stream if best quality not available
-                del self.streams_sorted[0]
-                stream_id = self.streams_sorted[0]['id']
-            else: break
-
-        if not kwargs['info_only']:
-            self.streams[stream_id]['src'] = ksegs
-    '''
-
-    def open_download_by_vid(self, client_id, vid, **kwargs):
-        """self, str, str, **kwargs->None
-
-        Arguments:
-        client_id:        An ID per client. For now we only know Acfun's
-                          such ID.
-
-        vid:              An video ID for each video, starts with "C".
-
-        kwargs['embsig']: Youku COOP's anti hotlinking.
-                          For Acfun, an API call must be done to Acfun's
-                          server, or the "playsign" of the content of sign_url
-                          shall be empty.
-
-        Misc:
-        Override the original one with VideoExtractor.
-
-        Author:
-        Most of the credit are to @ERioK, who gave his POC.
-
-        History:
-        Jul.28.2016 Youku COOP now have anti hotlinking via embsig. """
-        self.f_code_1 = '10ehfkbv'  #can be retrived by running r.translate with the keys and the list e
-        self.f_code_2 = 'msjv7h2b'
-
-        # as in VideoExtractor
-        self.url = None
-        self.vid = vid
-        self.name = "优酷开放平台 (Youku COOP)"
-
-        #A little bit of work before self.prepare
-
-        #Change as Jul.28.2016 Youku COOP updates its platform to add ant hotlinking
-        if kwargs['embsig']:
-            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}&embsig={embsig}".format(client_id = client_id, video_id = vid, embsig = kwargs['embsig'])
+        if 'dvd' in self.api_data:
+            al = self.api_data['dvd'].get('audiolang')
+            if al:
+                self.audiolang = al
+                for i in self.audiolang:
+                    i['url'] = 'http://v.youku.com/v_show/id_{}'.format(i['vid'])
+
+
+def youku_download_playlist_by_url(url, **kwargs):
+    video_page_pt = 'https?://v.youku.com/v_show/id_([A-Za-z0-9=]+)'
+    js_cb_pt = '\(({.+})\)'
+    if re.match(video_page_pt, url):
+        youku_obj = Youku()
+        youku_obj.url = url
+        youku_obj.prepare(**kwargs)
+        if youku_obj.video_list is None:
+            log.wtf('Cannot find video list for {}'.format(url))
         else:
-            sign_url = "https://api.youku.com/players/custom.json?client_id={client_id}&video_id={video_id}".format(client_id = client_id, video_id = vid)
-
-        playsign = json.loads(get_content(sign_url))['playsign']
-
-        #to be injected and replace ct10 and 12
-        api85_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=85&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
-        api86_url = 'http://play.youku.com/partner/get.json?cid={client_id}&vid={vid}&ct=86&sign={playsign}'.format(client_id = client_id, vid = vid, playsign = playsign)
-
-        self.prepare(api_url = api85_url, api12_url = api86_url, ctype = 86, **kwargs)
-
-        #exact copy from original VideoExtractor
-        if 'extractor_proxy' in kwargs and kwargs['extractor_proxy']:
-            unset_proxy()
-
-        try:
-            self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
-        except:
-            self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
-
-        self.extract(**kwargs)
-
-        self.download(**kwargs)
-
-site = Youku()
-download = site.download_by_url
-download_playlist = site.download_playlist_by_url
-
-youku_download_by_vid = site.download_by_vid
-youku_open_download_by_vid = site.open_download_by_vid
-# Used by: acfun.py bilibili.py miomio.py tudou.py
-# acfun has its own proxy and won't use it
-# miomio is dead
-# tudou doesn't use ct85 so open_download_by_vid is uesless now.
+            vid_list = [v['encodevid'] for v in youku_obj.video_list]
+            for v in vid_list:
+                youku_obj.download_by_vid(v, **kwargs)
+
+    elif re.match('https?://list.youku.com/show/id_', url):
+        # http://list.youku.com/show/id_z2ae8ee1c837b11e18195.html
+        # official playlist
+        page = get_content(url)
+        show_id = re.search(r'showid:"(\d+)"', page).group(1)
+        ep = 'http://list.youku.com/show/module?id={}&tab=showInfo&callback=jQuery'.format(show_id)
+        xhr_page = get_content(ep).replace('\/', '/').replace('\"', '"')
+        video_url = re.search(r'(v.youku.com/v_show/id_(?:[A-Za-z0-9=]+)\.html)', xhr_page).group(1)
+        youku_download_playlist_by_url('http://'+video_url, **kwargs)
+        return
+    elif re.match('https?://list.youku.com/albumlist/show/id_(\d+)\.html', url):
+        # http://list.youku.com/albumlist/show/id_2336634.html
+        # UGC playlist
+        list_id = re.search('https?://list.youku.com/albumlist/show/id_(\d+)\.html', url).group(1)
+        ep = 'http://list.youku.com/albumlist/items?id={}&page={}&size=20&ascending=1&callback=tuijsonp6'
+
+        first_u = ep.format(list_id, 1)
+        xhr_page = get_content(first_u)
+        json_data = json.loads(re.search(js_cb_pt, xhr_page).group(1))
+        video_cnt = json_data['data']['total']
+        xhr_html = json_data['html']
+        v_urls = re.findall(r'(v.youku.com/v_show/id_(?:[A-Za-z0-9=]+)\.html)', xhr_html)
+
+        if video_cnt > 20:
+            req_cnt = video_cnt // 20
+            for i in range(2, req_cnt+2):
+                req_u = ep.format(list_id, i)
+                xhr_page = get_content(req_u)
+                json_data = json.loads(re.search(js_cb_pt, xhr_page).group(1).replace('\/', '/'))
+                xhr_html = json_data['html']
+                page_videos = re.findall(r'(v.youku.com/v_show/id_(?:[A-Za-z0-9=]+)\.html)', xhr_html)
+                v_urls.extend(page_videos)
+        for u in v_urls[0::2]:
+            url = 'http://' + u
+            Youku().download_by_url(url, **kwargs)
+        return
+
+
+def youku_download_by_url(url, **kwargs):
+    Youku().download_by_url(url, **kwargs)
+
+
+def youku_download_by_vid(vid, **kwargs):
+    Youku().download_by_vid(vid, **kwargs)
+
+download = youku_download_by_url
+download_playlist = youku_download_playlist_by_url

From 5a8715fc96788b2446b090adcee370087b228d96 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 11 Aug 2017 16:27:37 +0800
Subject: [PATCH 0452/1225] [cntv]support multi stream types

---
 src/you_get/extractors/cntv.py | 76 ++++++++++++++++++++--------------
 1 file changed, 44 insertions(+), 32 deletions(-)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index bf926d2694..479481defe 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -1,55 +1,67 @@
 #!/usr/bin/env python
 
+import json
+import re
+
+from ..common import get_content, r1, match1, playlist_not_supported
+from ..extractor import VideoExtractor
+
 __all__ = ['cntv_download', 'cntv_download_by_id']
 
-from ..common import *
 
-import json
-import re
+class CNTV(VideoExtractor):
+    name = 'CNTV.com'
+    stream_types = [
+        {'id': '1', 'video_profile': '1280x720_2000kb/s', 'map_to': 'chapters4'},
+        {'id': '2', 'video_profile': '1280x720_1200kb/s', 'map_to': 'chapters3'},
+        {'id': '3', 'video_profile': '640x360_850kb/s', 'map_to': 'chapters2'},
+        {'id': '4', 'video_profile': '480x270_450kb/s', 'map_to': 'chapters'},
+        {'id': '5', 'video_profile': '320x180_200kb/s', 'map_to': 'lowChapters'},
+    ]
+
+    ep = 'http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid={}'
+
+    def __init__(self):
+        super().__init__()
+        self.api_data = None
+
+    def prepare(self, **kwargs):
+        self.api_data = json.loads(get_content(self.__class__.ep.format(self.vid)))
+        self.title = self.api_data['title']
+        for s in self.api_data['video']:
+            for st in self.__class__.stream_types:
+                if st['map_to'] == s:
+                    urls = self.api_data['video'][s]
+                    src = [u['url'] for u in urls]
+                    stream_data = dict(src=src, size=0, container='mp4', video_profile=st['video_profile'])
+                    self.streams[st['id']] = stream_data
+
+
+def cntv_download_by_id(rid, **kwargs):
+    CNTV().download_by_vid(rid, **kwargs)
 
 
-def cntv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
-    assert id
-    info = json.loads(get_html('http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=' + id))
-    title = title or info['title']
-    video = info['video']
-    alternatives = [x for x in video.keys() if x.endswith('hapters')]
-    #assert alternatives in (['chapters'], ['lowChapters', 'chapters'], ['chapters', 'lowChapters']), alternatives
-    chapters = video['chapters'] if 'chapters' in video else video['lowChapters']
-    urls = [x['url'] for x in chapters]
-    ext = r1(r'\.([^.]+)$', urls[0])
-    assert ext in ('flv', 'mp4')
-    size = 0
-    for url in urls:
-        _, _, temp = url_info(url)
-        size += temp
-
-    print_info(site_info, title, ext, size)
-    if not info_only:
-        # avoid corrupted files - don't merge
-        download_urls(urls, title, ext, size, output_dir = output_dir, merge = False)
-
-def cntv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+def cntv_download(url, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
-        id = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
+        rid = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
     elif re.match(r'http://tv\.cctv\.com/\d+/\d+/\d+/\w+.shtml', url):
-        id = r1(r'var guid = "(\w+)"', get_html(url))
+        rid = r1(r'var guid = "(\w+)"', get_content(url))
     elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
          re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \
          re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url): 
         page = get_content(url)
-        id = r1(r'videoCenterId","(\w+)"', page)
-        if id is None:
+        rid = r1(r'videoCenterId","(\w+)"', page)
+        if rid is None:
             guid = re.search(r'guid\s*=\s*"([0-9a-z]+)"', page).group(1)
-            id = guid
+            rid = guid
     elif re.match(r'http://xiyou.cntv.cn/v-[\w-]+\.html', url):
-        id = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)
+        rid = r1(r'http://xiyou.cntv.cn/v-([\w-]+)\.html', url)
     else:
         raise NotImplementedError(url)
 
-    cntv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
+    CNTV().download_by_vid(rid, **kwargs)
 
 site_info = "CNTV.com"
 download = cntv_download

From 7abb26c34b47e0e35d28596728a83d6c8d927683 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 12 Aug 2017 00:02:57 +0200
Subject: [PATCH 0453/1225] [miaopai] fix mp4 extension

---
 src/you_get/extractors/miaopai.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index ca543d15bc..6d9a79c719 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -24,8 +24,8 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
     if not title:
         title = fid
     title = title.replace('\n', '_')
-    type_, ext, size = url_info(url)
-    print_info(site_info, title, 'mp4', size)
+    ext, size = 'mp4', url_info(url)[2]
+    print_info(site_info, title, ext, size)
     if not info_only:
         download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 

From f99b068ee36ab109234e864501164566d999846b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 12 Aug 2017 11:26:02 +0800
Subject: [PATCH 0454/1225] [bilibili]index title could be described in CJK
 lang so it cannot be sorted at all

---
 src/you_get/extractors/bilibili.py | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 0cd217a069..4b801e6219 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -209,26 +209,24 @@ def bangumi_entry(self, **kwargs):
         bangumi_payment = bangumi_data.get('payment')
         if bangumi_payment and bangumi_payment['price'] != '0':
             log.w("It's a paid item")
-        ep_ids = collect_bangumi_epids(bangumi_data)
+        # ep_ids = collect_bangumi_epids(bangumi_data)
 
         frag = urllib.parse.urlparse(self.url).fragment
         if frag:
             episode_id = frag
         else:
             episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page)
-        cont = post_content('http://bangumi.bilibili.com/web_api/get_source', post_data=dict(episode_id=episode_id))
-        cid = json.loads(cont)['result']['cid']
+        # cont = post_content('http://bangumi.bilibili.com/web_api/get_source', post_data=dict(episode_id=episode_id))
+        # cid = json.loads(cont)['result']['cid']
         cont = get_content('http://bangumi.bilibili.com/web_api/episode/{}.json'.format(episode_id))
         ep_info = json.loads(cont)['result']['currentEpisode']
 
-        long_title = ep_info['longTitle']
-        aid = ep_info['avId']
-
-        idx = 0
-        while ep_ids[idx] != episode_id:
-            idx += 1
+        index_title = ep_info['indexTitle']
+        long_title = ep_info['longTitle'].strip()
+        cid = ep_info['danmaku']
 
-        self.title = '{} [{} {}]'.format(self.title, idx+1, long_title)
+        self.title = '{} [{} {}]'.format(self.title, index_title, long_title)
+        print(self.title)
         self.download_by_vid(cid, bangumi=True, **kwargs)
 
 
@@ -265,12 +263,8 @@ def fetch_sid(cid, aid):
     raise
 
 def collect_bangumi_epids(json_data):
-    eps = json_data['result']['episodes']
-    eps = sorted(eps, key=lambda item: float(item['index'].split('-')[0].split('+')[0]))
-    result = []
-    for ep in eps:
-        result.append(ep['episode_id'])
-    return result
+    eps = json_data['episodes'][::-1]
+    return [ep['episode_id'] for ep in eps]
 
 def get_bangumi_info(bangumi_id):
     BASE_URL = 'http://bangumi.bilibili.com/jsonp/seasoninfo/'
@@ -280,7 +274,7 @@ def get_bangumi_info(bangumi_id):
     season_data = season_data[len('seasonListCallback('):]
     season_data = season_data[: -1 * len(');')]
     json_data = json.loads(season_data)
-    return json_data
+    return json_data['result']
 
 def get_danmuku_xml(cid):
     return get_content('http://comment.bilibili.com/{}.xml'.format(cid))

From da916ec93f846386bdac7a4d7044c638a2eafd12 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 12 Aug 2017 16:00:14 +0800
Subject: [PATCH 0455/1225] [iqiyi]support zol.iqiyi; close #2256

---
 src/you_get/extractors/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index ee48942dec..12508fde86 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -131,10 +131,10 @@ def prepare(self, **kwargs):
             html = get_html(self.url)
             tvid = r1(r'#curid=(.+)_', self.url) or \
                    r1(r'tvid=([^&]+)', self.url) or \
-                   r1(r'data-player-tvid="([^"]+)"', html)
+                   r1(r'data-player-tvid="([^"]+)"', html) or r1(r'tv(?:i|I)d=(.+?)\&', html)
             videoid = r1(r'#curid=.+_(.*)$', self.url) or \
                       r1(r'vid=([^&]+)', self.url) or \
-                      r1(r'data-player-videoid="([^"]+)"', html)
+                      r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(.+?)\&', html)
             self.vid = (tvid, videoid)
             info_u = 'http://mixer.video.iqiyi.com/jp/mixin/videos/' + tvid
             mixin = get_content(info_u)

From b6b53d0d2ae01589b4197c08127fcb101a712617 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 12 Aug 2017 19:14:18 +0800
Subject: [PATCH 0456/1225] [embed vimeo]pass referer to VimeoExtractor

---
 src/you_get/extractors/embed.py | 2 +-
 src/you_get/extractors/vimeo.py | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 3bdb924ca1..8766497caf 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -82,7 +82,7 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
     urls = matchall(content, vimeo_embed_patters)
     for url in urls:
         found = True
-        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
 
     aids = matchall(content, bilibili_embed_patterns)
     for aid in aids:
diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index ece1be3a9a..c7d7b05754 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -16,10 +16,10 @@ def vimeo_download_by_channel(url, output_dir='.', merge=False, info_only=False,
     """str->None"""
     # https://vimeo.com/channels/464686
     channel_id = match1(url, r'http://vimeo.com/channels/(\w+)')
-    vimeo_download_by_channel_id(channel_id, output_dir, merge, info_only)
+    vimeo_download_by_channel_id(channel_id, output_dir, merge, info_only, **kwargs)
 
 #----------------------------------------------------------------------
-def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_only=False):
+def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_only=False, **kwargs):
     """str/int->None"""
     html = get_content('https://api.vimeo.com/channels/{channel_id}/videos?access_token={access_token}'.format(channel_id=channel_id, access_token=access_token))
     data = loads(html)
@@ -31,7 +31,7 @@ def vimeo_download_by_channel_id(channel_id, output_dir='.', merge=False, info_o
 
     for id in id_list:
         try:
-            vimeo_download_by_id(id, None, output_dir, merge, info_only)
+            vimeo_download_by_id(id, None, output_dir, merge, info_only, **kwargs)
         except urllib.error.URLError as e:
             log.w('{} failed with {}'.format(id, e))
 

From 5825564761154972cb1c0604931164811374bef4 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 12 Aug 2017 20:57:08 +0800
Subject: [PATCH 0457/1225] [embed]support iqiyi embed player; search iframe
 recursively

---
 src/you_get/extractors/embed.py     | 41 ++++++++++++++++++++++++++++-
 src/you_get/extractors/universal.py |  8 +++---
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 8766497caf..8b12a07cd4 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -1,5 +1,7 @@
 __all__ = ['embed_download']
 
+import urllib.parse
+
 from ..common import *
 
 from .bilibili import bilibili_download
@@ -12,6 +14,7 @@
 from .vimeo import vimeo_download_by_id
 from .yinyuetai import yinyuetai_download_by_id
 from .youku import youku_download_by_vid
+from . import iqiyi
 
 """
 refer to http://open.youku.com/tools
@@ -49,6 +52,15 @@
 bilibili_embed_patterns = [ 'static\.hdslb\.com/miniloader\.swf.*aid=(\d+)' ]
 
 
+'''
+http://open.iqiyi.com/lib/player.html
+'''
+iqiyi_patterns = [r'(?:\"|\')(https?://dispatcher\.video\.qiyi\.com\/disp\/shareplayer\.swf\?.+?)(?:\"|\')',
+                  r'(?:\"|\')(https?://open\.iqiyi\.com\/developer\/player_js\/coopPlayerIndex\.html\?.+?)(?:\"|\')']
+
+recur_limit = 3
+
+
 def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
     content = get_content(url, headers=fake_headers)
     found = False
@@ -90,8 +102,35 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         url = 'http://www.bilibili.com/video/av%s/' % aid
         bilibili_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
 
-    if not found:
+    iqiyi_urls = matchall(content, iqiyi_patterns)
+    for url in iqiyi_urls:
+        found = True
+        iqiyi.download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
+    if found:
+        return True
+
+    # Try harder, check all iframes
+    if 'recur_lv' not in kwargs or kwargs['recur_lv'] < recur_limit:
+        r = kwargs.get('recur_lv')
+        if r is None:
+            r = 1
+        else:
+            r += 1
+        iframes = matchall(content, [r'<iframe.+?src=(?:\"|\')(.+?)(?:\"|\')'])
+        for iframe in iframes:
+            if not iframe.startswith('http'):
+                src = urllib.parse.urljoin(url, iframe)
+            else:
+                src = iframe
+            found = embed_download(src, output_dir=output_dir, merge=merge, info_only=info_only, recur_lv=r, **kwargs)
+            if found:
+                return True
+
+    if not found and 'recur_lv' not in kwargs:
         raise NotImplementedError(url)
+    else:
+        return found
 
 site_info = "any.any"
 download = embed_download
diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 3168c48c6b..18385851e0 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -12,9 +12,11 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         content_type = get_head(url, headers=fake_headers, get_method='GET')['Content-Type']
     if content_type.startswith('text/html'):
         try:
-            embed_download(url, output_dir, merge=merge, info_only=info_only)
-        except: pass
-        else: return
+            embed_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        except Exception:
+            pass
+        else:
+            return
 
     domains = url.split('/')[2].split('.')
     if len(domains) > 2: domains = domains[1:]

From bcd8d74d73f13f310a92929c5654e208ef5140eb Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Sun, 13 Aug 2017 00:17:53 +0800
Subject: [PATCH 0458/1225] use argparse instead of getopt

see also https://github.com/soimort/you-get/issues/1181
---
 src/you_get/common.py | 376 +++++++++++++++++++++---------------------
 1 file changed, 185 insertions(+), 191 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e8d294dc79..d9ca96624b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -97,12 +97,10 @@
     'zhanqi'           : 'zhanqi',
 }
 
-import getopt
 import json
 import locale
 import logging
 import os
-import platform
 import re
 import socket
 import sys
@@ -110,6 +108,7 @@
 from urllib import request, parse, error
 from http import cookiejar
 from importlib import import_module
+import argparse
 
 from .version import __version__
 from .util import log, term
@@ -1056,216 +1055,211 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
-def script_main(script_name, download, download_playlist, **kwargs):
-    def version():
-        log.i('version %s, a tiny downloader that scrapes the web.'
-              % get_version(kwargs['repo_path']
-            if 'repo_path' in kwargs else __version__))
+def load_cookies(cookiefile):
+    global cookies
+    try:
+        cookies = cookiejar.MozillaCookieJar(a)
+        cookies.load()
+    except Exception:
+        import sqlite3
+        cookies = cookiejar.MozillaCookieJar()
+        con = sqlite3.connect(a)
+        cur = con.cursor()
+        try:
+            cur.execute("""SELECT host, path, isSecure, expiry, name, value
+                        FROM moz_cookies""")
+            for item in cur.fetchall():
+                c = cookiejar.Cookie(
+                    0, item[4], item[5], None, False, item[0],
+                    item[0].startswith('.'), item[0].startswith('.'),
+                    item[1], False, item[2], item[3], item[3]=="", None,
+                    None, {},
+                )
+                cookies.set_cookie(c)
+        except Exception:
+            pass
+        # TODO: Chromium Cookies
+        # SELECT host_key, path, secure, expires_utc, name, encrypted_value
+        # FROM cookies
+        # http://n8henrie.com/2013/11/use-chromes-cookies-for-easier-downloading-with-python-requests/
 
+def set_socks_proxy(proxy):
+    try:
+        import socks
+        socks_proxy_addrs = socks_proxy.split(':')
+        socks.set_default_proxy(socks.SOCKS5,
+                                socks_proxy_addrs[0],
+                                int(socks_proxy_addrs[1]))
+        socket.socket = socks.socksocket
+        def getaddrinfo(*args):
+            return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+        socket.getaddrinfo = getaddrinfo
+    except ImportError:
+        log.w('Error importing PySocks library, socks proxy ignored.'
+            'In order to use use socks proxy, please install PySocks.')
+
+def script_main(download, download_playlist, **kwargs):
     logging.basicConfig(format='[%(levelname)s] %(message)s')
 
-    help = 'Usage: %s [OPTION]... [URL]...\n\n' % script_name
-    help += '''Startup options:
-    -V | --version                      Print version and exit.
-    -h | --help                         Print help and exit.
-    \n'''
-    help += '''Dry-run options: (no actual downloading)
-    -i | --info                         Print extracted information.
-    -u | --url                          Print extracted information with URLs.
-         --json                         Print extracted URLs in JSON format.
-    \n'''
-    help += '''Download options:
-    -n | --no-merge                     Do not merge video parts.
-         --no-caption                   Do not download captions.
-                                        (subtitles, lyrics, danmaku, ...)
-    -f | --force                        Force overwriting existed files.
-    -F | --format <STREAM_ID>           Set video format to STREAM_ID.
-    -O | --output-filename <FILE>       Set output filename.
-    -o | --output-dir <PATH>            Set output directory.
-    -p | --player <PLAYER [OPTIONS]>    Stream extracted URL to a PLAYER.
-    -c | --cookies <COOKIES_FILE>       Load cookies.txt or cookies.sqlite.
-    -x | --http-proxy <HOST:PORT>       Use an HTTP proxy for downloading.
-    -y | --extractor-proxy <HOST:PORT>  Use an HTTP proxy for extracting only.
-         --no-proxy                     Never use a proxy.
-    -s | --socks-proxy <HOST:PORT>      Use an SOCKS5 proxy for downloading.
-    -t | --timeout <SECONDS>            Set socket timeout.
-    -d | --debug                        Show traceback and other debug info.
-    -I | --input-file                   Read non-playlist urls from file.
-    -P | --password <PASSWORD>          Set video visit password to PASSWORD.
-    '''
-
-    short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:P:'
-    opts = ['version', 'help', 'force', 'info', 'url', 'cookies', 'no-caption', 'no-merge', 'no-proxy', 'debug', 'json', 'format=', 'stream=', 'itag=', 'output-filename=', 'output-dir=', 'player=', 'http-proxy=', 'socks-proxy=', 'extractor-proxy=', 'lang=', 'timeout=', 'input-file=', 'password=']
-#dead code? download_playlist is a function and always True
-#if download_playlist:
-    short_opts = 'l' + short_opts
-    opts = ['playlist'] + opts
+    def print_version():
+        log.i('version %s, a tiny downloader that scrapes the web.'
+                % get_version(kwargs['repo_path']
+            if 'repo_path' in kwargs else __version__))
 
-    try:
-        opts, args = getopt.gnu_getopt(sys.argv[1:], short_opts, opts)
-    except getopt.GetoptError as err:
-        log.e(err)
-        log.e("try 'you-get --help' for more options")
-        sys.exit(2)
+    parser = argparse.ArgumentParser(
+        prog='you-get',
+        usage='you-get [OPTION]... URL...',
+        description='A tiny downloader that scrapes the web',
+        add_help=False,
+    )
+    parser.add_argument('-V', '--version', action='store_true',
+                        help='Print version and exit')
+    parser.add_argument('-h', '--help', action='store_true',
+                        help='Print this help message and exit')
+
+    dry_run_grp = parser.add_argument_group('Dry-run options', '(no actual downloading)')
+    dry_run_grp = dry_run_grp.add_mutually_exclusive_group()
+    dry_run_grp.add_argument('-i', '--info', action='store_true',
+                        help='Print extracted information')
+    dry_run_grp.add_argument('-u', '--url', action='store_true',
+                        help='Print extracted information with URLs')
+    dry_run_grp.add_argument('--json', action='store_true',
+                        help='Print extracted URLs in JSON format')
+
+    download_grp = parser.add_argument_group('Download options')
+    download_grp.add_argument('-n', '--no-merge', action='store_true', default=False,
+                        help='Do not merge video parts')
+    download_grp.add_argument('--no-caption', action='store_true',
+                        help='Do not download captions (subtitles, lyrics, danmaku, ...)')
+    download_grp.add_argument('-f', '--force', action='store_true', default=False,
+                        help='Force overwriting existing files')
+    download_grp.add_argument('-F', '--format', metavar='STREAM_ID',
+                        help='Set video format to STREAM_ID')
+    download_grp.add_argument('-O', '--output-filename', metavar='FILE',
+                        help='Set output filename')
+    download_grp.add_argument('-o', '--output-dir', metavar='DIR', default='.',
+                        help='Set output directory')
+    download_grp.add_argument('-p', '--player', metavar='PLAYER',
+                        help='Stream extracted URL to a PLAYER')
+    download_grp.add_argument('-c', '--cookies', metavar='COOKIES_FILE',
+                        help='Load cookies.txt or cookies.sqlite')
+    download_grp.add_argument('-t', '--timeout', metavar='SECONDS', type=int, default=600,
+                        help='Set socket timeout')
+    download_grp.add_argument('-d', '--debug', action='store_true',
+                        help='Show traceback and other debug info')
+    download_grp.add_argument('-I', '--input-file', metavar='FILE', type=argparse.FileType('r'),
+                        help='Read non-playlist URLs from FILE')
+    download_grp.add_argument('-P', '--password',
+                        help='Set video visit password to PASSWORD')
+    download_grp.add_argument('-l', '--playlist', action='store_true',
+                        help='Prefer to download a playlist')
+
+    proxy_grp = parser.add_argument_group('Proxy options')
+    proxy_grp = proxy_grp.add_mutually_exclusive_group()
+    proxy_grp.add_argument('-x', '--http-proxy', metavar='HOST:PORT',
+                        help='Use an HTTP proxy for downloading')
+    proxy_grp.add_argument('-y', '--extractor-proxy', metavar='HOST:PORT',
+                        help='Use an HTTP proxy for extracting only')
+    proxy_grp.add_argument('--no-proxy', action='store_true',
+                        help='Never use a proxy')
+    proxy_grp.add_argument('-s', '--socks-proxy', metavar='HOST:PORT',
+                        help='Use an SOCKS5 proxy for downloading')
+
+    download_grp.add_argument('--stream',
+                        help=argparse.SUPPRESS)
+    download_grp.add_argument('--itag',
+                        help=argparse.SUPPRESS)
+
+    parser.add_argument('URL', nargs='*',
+                        help=argparse.SUPPRESS)
+
+    args = parser.parse_args()
+
+    if args.help:
+        print_version()
+        parser.print_help()
+        sys.exit()
+    if args.version:
+        print_version()
+        sys.exit()
+
+    if args.debug:
+        # Set level of root logger to DEBUG
+        logging.getLogger().setLevel(logging.DEBUG)
 
     global force
     global dry_run
     global json_output
     global player
     global extractor_proxy
-    global cookies
     global output_filename
 
-    info_only = False
-    playlist = False
+    output_filename = args.output_filename
+    extractor_proxy = args.extractor_proxy
+
+    info_only = args.info
+    if args.url:
+        dry_run = True
+    if args.json:
+        json_output = True
+        # to fix extractors not use VideoExtractor
+        dry_run = True
+        info_only = False
+
+    if args.cookies:
+        load_cookies(args.cookies)
+
     caption = True
-    merge = True
-    stream_id = None
-    lang = None
-    output_dir = '.'
-    proxy = None
-    socks_proxy = None
-    extractor_proxy = None
-    traceback = False
-    timeout = 600
-    urls_from_file = []
-    password = None
-
-    for o, a in opts:
-        if o in ('-V', '--version'):
-            version()
-            sys.exit()
-        elif o in ('-h', '--help'):
-            version()
-            print(help)
-            sys.exit()
-        elif o in ('-f', '--force'):
-            force = True
-        elif o in ('-i', '--info'):
-            info_only = True
-        elif o in ('-u', '--url'):
-            dry_run = True
-        elif o in ('--json', ):
-            json_output = True
-            # to fix extractors not use VideoExtractor
-            dry_run = True
-            info_only = False
-        elif o in ('-c', '--cookies'):
-            try:
-                cookies = cookiejar.MozillaCookieJar(a)
-                cookies.load()
-            except:
-                import sqlite3
-                cookies = cookiejar.MozillaCookieJar()
-                con = sqlite3.connect(a)
-                cur = con.cursor()
-                try:
-                    cur.execute("SELECT host, path, isSecure, expiry, name, value FROM moz_cookies")
-                    for item in cur.fetchall():
-                        c = cookiejar.Cookie(0, item[4], item[5],
-                                             None, False,
-                                             item[0],
-                                             item[0].startswith('.'),
-                                             item[0].startswith('.'),
-                                             item[1], False,
-                                             item[2],
-                                             item[3], item[3]=="",
-                                             None, None, {})
-                        cookies.set_cookie(c)
-                except: pass
-                # TODO: Chromium Cookies
-                # SELECT host_key, path, secure, expires_utc, name, encrypted_value FROM cookies
-                # http://n8henrie.com/2013/11/use-chromes-cookies-for-easier-downloading-with-python-requests/
-
-        elif o in ('-l', '--playlist'):
-            playlist = True
-        elif o in ('--no-caption',):
-            caption = False
-        elif o in ('-n', '--no-merge'):
-            merge = False
-        elif o in ('--no-proxy',):
-            proxy = ''
-        elif o in ('-d', '--debug'):
-            traceback = True
-            # Set level of root logger to DEBUG
-            logging.getLogger().setLevel(logging.DEBUG)
-        elif o in ('-F', '--format', '--stream', '--itag'):
-            stream_id = a
-        elif o in ('-O', '--output-filename'):
-            output_filename = a
-        elif o in ('-o', '--output-dir'):
-            output_dir = a
-        elif o in ('-p', '--player'):
-            player = a
-            caption = False
-        elif o in ('-x', '--http-proxy'):
-            proxy = a
-        elif o in ('-s', '--socks-proxy'):
-            socks_proxy = a
-        elif o in ('-y', '--extractor-proxy'):
-            extractor_proxy = a
-        elif o in ('--lang',):
-            lang = a
-        elif o in ('-t', '--timeout'):
-            timeout = int(a)
-        elif o in ('-P', '--password',):
-            password = a
-        elif o in ('-I', '--input-file'):
-            logging.debug('you are trying to load urls from {}'.format(a))
-            if playlist:
-                log.e("reading playlist from a file is unsupported and won't make your life easier")
-                sys.exit(2)
-            with open(a, 'r') as input_file:
-                for line in input_file:
-                    url = line.strip()
-                    urls_from_file.append(url)
-        else:
-            log.e("try 'you-get --help' for more options")
+    stream_id = args.format or args.stream or args.itag
+    if args.no_caption:
+        caption = False
+    if args.player:
+        player = args.player
+        caption = False
+
+    if args.no_proxy:
+        set_http_proxy('')
+    else:
+        set_http_proxy(args.http_proxy)
+    if args.socks_proxy:
+        set_socks_proxy(args.socks_proxy)
+
+    URLs = []
+    if args.input_file:
+        logging.debug('you are trying to load urls from %s', args.input_file)
+        if args.playlist:
+            log.e("reading playlist from a file is unsupported and won't make your life easier")
             sys.exit(2)
-    if not args and not urls_from_file:
-        print(help)
-        sys.exit()
-    args.extend(urls_from_file)
+        URLs.extend(args.input_file.read().splitlines())
+        args.input_file.close()
+    URLs.extend(args.URL)
 
-    if (socks_proxy):
-        try:
-            import socket
-            import socks
-            socks_proxy_addrs = socks_proxy.split(':')
-            socks.set_default_proxy(socks.SOCKS5,
-                                    socks_proxy_addrs[0],
-                                    int(socks_proxy_addrs[1]))
-            socket.socket = socks.socksocket
-            def getaddrinfo(*args):
-                return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
-            socket.getaddrinfo = getaddrinfo
-        except ImportError:
-            log.w('Error importing PySocks library, socks proxy ignored.'
-                'In order to use use socks proxy, please install PySocks.')
-    else:
-        import socket
-        set_http_proxy(proxy)
+    if not URLs:
+        parser.print_help()
+        sys.exit()
 
-    socket.setdefaulttimeout(timeout)
+    socket.setdefaulttimeout(args.timeout)
 
     try:
+        extra = {}
+        if extractor_proxy:
+            extra['extractor_proxy'] = extractor_proxy
         if stream_id:
-            if not extractor_proxy:
-                download_main(download, download_playlist, args, playlist, stream_id=stream_id, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
-            else:
-                download_main(download, download_playlist, args, playlist, stream_id=stream_id, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
-        else:
-            if not extractor_proxy:
-                download_main(download, download_playlist, args, playlist, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
-            else:
-                download_main(download, download_playlist, args, playlist, extractor_proxy=extractor_proxy, output_dir=output_dir, merge=merge, info_only=info_only, json_output=json_output, caption=caption)
+            extra['stream_id'] = stream_id
+        download_main(
+            download, download_playlist,
+            URLs, args.playlist,
+            stream_id=stream_id, output_dir=args.output_dir, merge=not args.no_merge,
+            info_only=info_only, json_output=json_output, caption=caption,
+            **extra
+        )
     except KeyboardInterrupt:
-        if traceback:
+        if args.debug:
             raise
         else:
             sys.exit(1)
     except UnicodeEncodeError:
-        if traceback:
+        if args.debug:
             raise
         log.e('[error] oops, the current environment does not seem to support Unicode.')
         log.e('please set it to a UTF-8-aware locale first,')
@@ -1275,7 +1269,7 @@ def getaddrinfo(*args):
         log.e('    (Linux)      $ LC_CTYPE=en_US.UTF-8')
         sys.exit(1)
     except Exception:
-        if not traceback:
+        if not args.debug:
             log.e('[error] oops, something went wrong.')
             log.e('don\'t panic, c\'est la vie. please try the following steps:')
             log.e('  (1) Rule out any network problem.')
@@ -1286,7 +1280,7 @@ def getaddrinfo(*args):
             log.e('  (4) Run the command with \'--debug\' option,')
             log.e('      and report this issue with the full output.')
         else:
-            version()
+            print_version()
             log.i(args)
             raise
         sys.exit(1)
@@ -1349,4 +1343,4 @@ def any_download_playlist(url, **kwargs):
     m.download_playlist(url, **kwargs)
 
 def main(**kwargs):
-    script_main('you-get', any_download, any_download_playlist, **kwargs)
+    script_main(any_download, any_download_playlist, **kwargs)

From 06463a8880ce06af03f235d91584865a1c5de8c9 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 13 Aug 2017 00:53:49 +0800
Subject: [PATCH 0459/1225] [vk]support user video list; fix #1311

---
 src/you_get/extractors/vk.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index 98f3471b85..30fd2cac3e 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -22,6 +22,19 @@ def get_video_info(url):
     return url, title, ext, size
 
 
+def get_video_from_user_videolist(url):
+    ep = 'https://vk.com/al_video.php'
+    to_post = dict(act='show', al=1, module='direct', video=re.search(r'video(\d+_\d+)', url).group(1))
+    page = post_content(ep, post_data=to_post)
+    video_pt = r'<source src="(.+?)" type="video\/mp4"'
+    url = re.search(video_pt, page).group(1)
+    title = re.search(r'<div class="mv_title".+?>(.+?)</div>', page).group(1)
+    mime, ext, size = url_info(url)
+    print_info(site_info, title, mime, size)
+
+    return url, title, ext, size
+
+
 def get_image_info(url):
     image_page = get_content(url)
     # used for title - vk page owner
@@ -43,6 +56,8 @@ def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=Fal
         link, title, ext, size = get_video_info(url)
     elif re.match(r'(.+)vk\.com\/photo(.+)', url):
         link, title, ext, size = get_image_info(url)
+    elif re.search(r'vk\.com\/video\d+_\d+', url):
+        get_video_from_user_videolist(url)
     else:
         raise NotImplementedError('Nothing to download here')
 

From b4ce413dea2383ffd3a33c90e590bf2b491e47bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 12 Aug 2017 21:00:15 +0200
Subject: [PATCH 0460/1225] [vk] get things done

---
 src/you_get/extractors/vk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/vk.py b/src/you_get/extractors/vk.py
index 30fd2cac3e..299fd89a7f 100644
--- a/src/you_get/extractors/vk.py
+++ b/src/you_get/extractors/vk.py
@@ -57,7 +57,7 @@ def vk_download(url, output_dir='.', stream_type=None, merge=True, info_only=Fal
     elif re.match(r'(.+)vk\.com\/photo(.+)', url):
         link, title, ext, size = get_image_info(url)
     elif re.search(r'vk\.com\/video\d+_\d+', url):
-        get_video_from_user_videolist(url)
+        link, title, ext, size = get_video_from_user_videolist(url)
     else:
         raise NotImplementedError('Nothing to download here')
 

From 699ed7e50117882bcd016ed1a5c1b30e37ac217e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 13 Aug 2017 19:55:19 +0800
Subject: [PATCH 0461/1225] [common]catch HTTPError in urlopen_with_retry

---
 src/you_get/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index e8d294dc79..5484fe80f1 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -339,11 +339,14 @@ def get_location(url):
     return response.geturl()
 
 def urlopen_with_retry(*args, **kwargs):
-    for i in range(10):
+    for i in range(2):
         try:
             return request.urlopen(*args, **kwargs)
         except socket.timeout:
             logging.debug('request attempt %s timeout' % str(i + 1))
+# try to tackle youku CDN fails
+        except error.HTTPError as http_error:
+            logging.debug('HTTP Error with code{}'.format(http_error.code))
 
 def get_content(url, headers={}, decoded=True):
     """Gets the content of a URL via sending a HTTP GET request.

From c15b8940c31416db6816062da2c18084ab35ca18 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 14 Aug 2017 16:05:24 +0800
Subject: [PATCH 0462/1225] [qq qq_egame]support qq egame

---
 src/you_get/extractors/qq.py       |  5 ++++
 src/you_get/extractors/qq_egame.py | 41 ++++++++++++++++++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 src/you_get/extractors/qq_egame.py

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index fedaf5f9fe..a5bb22768f 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -86,6 +86,11 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
+    if re.match(r'https?://egame.qq.com/live\?anchorid=(\d+)', url):
+        from . import qq_egame
+        qq_egame.qq_egame_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
+
     if 'kg.qq.com' in url or 'kg2.qq.com' in url:
         shareid = url.split('?s=')[-1]
         caption = kwargs['caption']
diff --git a/src/you_get/extractors/qq_egame.py b/src/you_get/extractors/qq_egame.py
new file mode 100644
index 0000000000..4ec36ef20d
--- /dev/null
+++ b/src/you_get/extractors/qq_egame.py
@@ -0,0 +1,41 @@
+import re
+import json
+
+from ..common import get_content
+from ..extractors import VideoExtractor
+from ..util import log
+from ..util.strings import unescape_html
+
+__all__ = ['qq_egame_download']
+
+
+class QQEgame(VideoExtractor):
+    stream_types = [
+        {'id': 'original', 'video_profile': '0', 'container': 'flv'},
+        {'id': '900', 'video_profile': '900kb/s', 'container': 'flv'},
+        {'id': '550', 'video_profile': '550kb/s', 'container': 'flv'}
+    ]
+    name = 'QQEgame'
+
+    def prepare(self, **kwargs):
+        page = get_content(self.url)
+        server_data = re.search(r'serverData\s*=\s*({.+?});', page)
+        if server_data is None:
+            log.wtf('cannot find server_data')
+        json_data = json.loads(server_data.group(1))
+        live_info = json_data['liveInfo']['data']
+        self.title = '{}_{}'.format(live_info['profileInfo']['nickName'], live_info['videoInfo']['title'])
+        for exsited_stream in live_info['videoInfo']['streamInfos']:
+            for s in self.__class__.stream_types:
+                if re.search(r'(\d+)', s['video_profile']).group(1) == exsited_stream['bitrate']:
+                    current_stream_id = s['id']
+                    stream_info = dict(src=[unescape_html(exsited_stream['playUrl'])])
+                    stream_info['video_profile'] = exsited_stream['desc']
+                    stream_info['container'] = s['container']
+                    stream_info['size'] = float('inf')
+                    self.streams[current_stream_id] = stream_info
+
+
+def qq_egame_download(url, **kwargs):
+    QQEgame().download_by_url(url, **kwargs)
+    # url dispatching has been done in qq.py

From 1950caf082e7439adaa231343108b4c186670312 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Mon, 14 Aug 2017 22:19:15 +0800
Subject: [PATCH 0463/1225] [zhanqi]new video pattern; fix #2272

---
 src/you_get/extractors/zhanqi.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/zhanqi.py b/src/you_get/extractors/zhanqi.py
index d0bbddb65a..8daf341370 100644
--- a/src/you_get/extractors/zhanqi.py
+++ b/src/you_get/extractors/zhanqi.py
@@ -10,11 +10,12 @@
 def zhanqi_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     path = urlparse(url).path[1:]
 
-    if not path.startswith('videos'): #url = "https://www.zhanqi.tv/huashan?param_s=1_0.2.0"
+    if not (path.startswith('videos') or path.startswith('v2/videos')): #url = "https://www.zhanqi.tv/huashan?param_s=1_0.2.0"
         path_list = path.split('/')
         room_id = path_list[1] if path_list[0] == 'topic' else path_list[0]
         zhanqi_live(room_id, merge=merge, output_dir=output_dir, info_only=info_only, **kwargs)
     else: #url = 'https://www.zhanqi.tv/videos/Lyingman/2017/01/182308.html'
+        # https://www.zhanqi.tv/v2/videos/215593.html
         video_id = path.split('.')[0].split('/')[-1]
         zhanqi_video(video_id, merge=merge, output_dir=output_dir, info_only=info_only, **kwargs)
 

From ed0451016c515417cd201ebd53405aa96d44501e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 15 Aug 2017 21:50:34 +0800
Subject: [PATCH 0464/1225] [naver]rewrite

---
 src/you_get/extractors/naver.py | 80 +++++++++++++++------------------
 1 file changed, 35 insertions(+), 45 deletions(-)

diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
index 7f7d554875..d79e5245e0 100644
--- a/src/you_get/extractors/naver.py
+++ b/src/you_get/extractors/naver.py
@@ -1,48 +1,38 @@
 #!/usr/bin/env python
 
-__all__ = ['naver_download']
-import urllib.request, urllib.parse
-from ..common import *
-
-def naver_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-
-	assert re.search(r'http://tvcast.naver.com/v/', url), "URL is not supported"
-
-	html = get_html(url)
-	contentid = re.search(r'var rmcPlayer = new nhn.rmcnmv.RMCVideoPlayer\("(.+?)", "(.+?)"',html)
-	videoid = contentid.group(1)
-	inkey = contentid.group(2)
-	assert videoid
-	assert inkey
-	info_key = urllib.parse.urlencode({'vid': videoid, 'inKey': inkey, })
-	down_key = urllib.parse.urlencode({'masterVid': videoid,'protocol': 'p2p','inKey': inkey, })
-	inf_xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/videoInfo.nhn?%s' % info_key )
-
-	from xml.dom.minidom import parseString
-	doc_info = parseString(inf_xml)
-	Subject = doc_info.getElementsByTagName('Subject')[0].firstChild
-	title = Subject.data
-	assert title
-
-	xml = get_html('http://serviceapi.rmcnmv.naver.com/flash/playableEncodingOption.nhn?%s' % down_key )
-	doc = parseString(xml)
-
-	encodingoptions = doc.getElementsByTagName('EncodingOption')
-	old_height = doc.getElementsByTagName('height')[0]
-	real_url= ''
-	#to download the highest resolution one,
-	for node in encodingoptions:
-		new_height = node.getElementsByTagName('height')[0]
-		domain_node = node.getElementsByTagName('Domain')[0]
-		uri_node = node.getElementsByTagName('uri')[0]
-		if int(new_height.firstChild.data) > int (old_height.firstChild.data):
-			real_url= domain_node.firstChild.data+ '/' +uri_node.firstChild.data
-
-	type, ext, size = url_info(real_url)
-	print_info(site_info, title, type, size)
-	if not info_only:
-		download_urls([real_url], title, ext, size, output_dir, merge = merge)
-
-site_info = "tvcast.naver.com"
-download = naver_download
+import urllib.request
+import urllib.parse
+import json
+import re
+
+from ..util import log
+from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size
+
+__all__ = ['naver_download_by_url']
+
+
+def naver_download_by_url(url, info_only=False, **kwargs):
+    ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'
+    page = get_content(url)
+    og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1)
+    params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
+    vid = params_dict['vid'][0]
+    key = params_dict['outKey'][0]
+    meta_str = get_content(ep.format(vid, key))
+    meta_json = json.loads(meta_str)
+    if 'errorCode' in meta_json:
+        log.wtf(meta_json['errorCode'])
+    title = meta_json['meta']['subject']
+    videos = meta_json['videos']['list']
+    video_list = sorted(videos, key=lambda video: video['encodingOption']['width'])
+    video_url = video_list[-1]['source']
+    # size = video_list[-1]['size']
+    # result wrong size
+    size = url_size(video_url)
+    print_info(site_info, title, 'mp4', size)
+    if not info_only:
+        download_urls([video_url], title, 'mp4', size, **kwargs)
+
+site_info = "naver.com"
+download = naver_download_by_url
 download_playlist = playlist_not_supported('naver')

From 6b76acd226ac5e24690e828f588e2402cf2a2288 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 15 Aug 2017 21:59:52 +0800
Subject: [PATCH 0465/1225] [dailymotion]fix #2275

---
 src/you_get/extractors/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index bbfd22fd5c..682d8e4dc1 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -10,7 +10,7 @@ def rebuilt_url(url):
     aid = path.split('/')[-1].split('_')[0]
     return 'http://www.dailymotion.com/embed/video/{}?autoplay=1'.format(aid)
 
-def dailymotion_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+def dailymotion_download(url, info_only=False, **kwargs):
     """Downloads Dailymotion videos by URL.
     """
 
@@ -32,7 +32,7 @@ def dailymotion_download(url, output_dir='.', merge=True, info_only=False, **kwa
 
     print_info(site_info, title, mime, size)
     if not info_only:
-        download_urls(real_url, title, ext, output_dir=output_dir, merge=merge)
+        download_urls([real_url], title, ext, size, **kwargs)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From ba864441a625fd6e315cabfd33bb62c61aa5803c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 15 Aug 2017 22:21:04 +0800
Subject: [PATCH 0466/1225] comment the WIP code to silent lint

---
 src/you_get/cli_wrapper/player/__main__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/cli_wrapper/player/__main__.py b/src/you_get/cli_wrapper/player/__main__.py
index 8d4958b914..09f4d42d6f 100644
--- a/src/you_get/cli_wrapper/player/__main__.py
+++ b/src/you_get/cli_wrapper/player/__main__.py
@@ -1,7 +1,9 @@
 #!/usr/bin/env python
 
+''' WIP
 def main():
     script_main('you-get', any_download, any_download_playlist)
 
 if __name__ == "__main__":
     main()
+'''

From e32e8c56ad8e6322e38c60c0ad4278e6535e1838 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 16 Aug 2017 16:26:25 +0800
Subject: [PATCH 0467/1225] [common]fix mutable default parameter in url_save;
 fix #2278

---
 src/you_get/common.py | 27 ++++++++++++++++-----------
 1 file changed, 16 insertions(+), 11 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5484fe80f1..52dbb74988 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -522,11 +522,13 @@ def url_locations(urls, faker = False, headers = {}):
         locations.append(response.url)
     return locations
 
-def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, headers = {}, timeout = None, **kwargs):
-#When a referer specified with param refer, the key must be 'Referer' for the hack here
+
+def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers=None, timeout=None, **kwargs):
+    tmp_headers = headers.copy() if headers is not None else {}
+# When a referer specified with param refer, the key must be 'Referer' for the hack here
     if refer is not None:
-        headers['Referer'] = refer
-    file_size = url_size(url, faker = faker, headers = headers)
+        tmp_headers['Referer'] = refer
+    file_size = url_size(url, faker=faker, headers=tmp_headers)
 
     if os.path.exists(filepath):
         if not force and file_size == os.path.getsize(filepath):
@@ -560,20 +562,23 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
 
     if received < file_size:
         if faker:
-            headers = fake_headers
+            tmp_headers = fake_headers
+        '''
+        if parameter headers passed in, we have it copied as tmp_header
         elif headers:
             headers = headers
         else:
             headers = {}
+        '''
         if received:
-            headers['Range'] = 'bytes=' + str(received) + '-'
+            tmp_headers['Range'] = 'bytes=' + str(received) + '-'
         if refer:
-            headers['Referer'] = refer
+            tmp_headers['Referer'] = refer
 
         if timeout:
-            response = urlopen_with_retry(request.Request(url, headers=headers), timeout=timeout)
+            response = urlopen_with_retry(request.Request(url, headers=tmp_headers), timeout=timeout)
         else:
-            response = urlopen_with_retry(request.Request(url, headers=headers))
+            response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
         try:
             range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
             end_length = int(response.headers['content-range'][6:].split('/')[1])
@@ -595,8 +600,8 @@ def url_save(url, filepath, bar, refer = None, is_part = False, faker = False, h
                     if received == file_size: # Download finished
                         break
                     else: # Unexpected termination. Retry request
-                        headers['Range'] = 'bytes=' + str(received) + '-'
-                        response = urlopen_with_retry(request.Request(url, headers=headers))
+                        tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+                        response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
                 output.write(buffer)
                 received += len(buffer)
                 if bar:

From 58f778e4f61d713674b23add2bff5049a97a3e60 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 17 Aug 2017 13:04:09 +0800
Subject: [PATCH 0468/1225] [youku]change host of cdn_url to aliCDN

---
 src/you_get/extractors/youku.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 95cab6b2ee..c65bf2dfaa 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -39,6 +39,7 @@ def quote_cna(val):
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
     mobile_ua = 'Mozilla/5.0 (iPad; CPU OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Mobile/14B100'
+    dispatcher_url = 'vali.cp31.ott.cibntv.net'
 
     # Last updated: 2015-11-24
     stream_types = [
@@ -89,6 +90,19 @@ def youku_ups(self):
             if 'list' in self.api_data['videos']:
                 self.video_list = self.api_data['videos']['list']
 
+    @classmethod
+    def change_cdn(cls, url):
+        # if the cnd_url starts with an ip addr, it should be youku's old CDN
+        # which rejects http requests randomly with status code > 400
+        # change it to the dispatcher of aliCDN can do better
+        # at least a little more recoverable from HTTP 403
+        if cls.dispatcher_url in url:
+            return url
+        else:
+            url_seg_list = list(urllib.parse.urlsplit(url))
+            url_seg_list[1] = cls.dispatcher_url
+            return urllib.parse.urlunsplit(url_seg_list)
+
     def get_vid_from_url(self):
         # It's unreliable. check #1633
         b64p = r'([a-zA-Z0-9=]+)'
@@ -179,7 +193,7 @@ def prepare(self, **kwargs):
                     src = []
                     for seg in stream['segs']:
                         if seg.get('cdn_url'):
-                            src.append(seg['cdn_url'])
+                            src.append(self.__class__.change_cdn(seg['cdn_url']))
                         else:
                             is_preview = True
                     self.streams[stream_id]['src'] = src
@@ -191,7 +205,7 @@ def prepare(self, **kwargs):
                     src = []
                     for seg in stream['segs']:
                         if seg.get('cdn_url'):
-                            src.append(seg['cdn_url'])
+                            src.append(self.__class__.change_cdn(seg['cdn_url']))
                         else:
                             is_preview = True
                     self.streams[stream_id]['src'].extend(src)

From 170d7f0c24aff6da490514b1a0e85e8edbc4a35b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 18 Aug 2017 00:40:01 +0800
Subject: [PATCH 0469/1225] [youku]do not change the url if it starts with
 k.youku.com

---
 src/you_get/extractors/youku.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index c65bf2dfaa..863573d742 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -98,6 +98,8 @@ def change_cdn(cls, url):
         # at least a little more recoverable from HTTP 403
         if cls.dispatcher_url in url:
             return url
+        elif 'k.youku.com' in url:
+            return url
         else:
             url_seg_list = list(urllib.parse.urlsplit(url))
             url_seg_list[1] = cls.dispatcher_url

From eaa692aa35a00099f0a4a0f79daaf5b021779ae6 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 18 Aug 2017 04:01:53 +0800
Subject: [PATCH 0470/1225] [icourses]for videos larger than 330MiB seek must
 be on 15MiB boundary; fix #2283

---
 src/you_get/extractors/icourses.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index c6321a6e26..0140281a26 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -42,7 +42,7 @@ def icourses_download(url, output_dir='.', **kwargs):
         real_url = icourses_parser.update_url(0)
         headers = fake_headers.copy()
         headers['Referer'] = url
-        download_urls_icourses(real_url, title, 'flv',total_size=size, output_dir=output_dir, max_size=15000000, dyn_callback=icourses_parser.update_url)
+        download_urls_icourses(real_url, title, 'flv',total_size=size, output_dir=output_dir, max_size=15728640, dyn_callback=icourses_parser.update_url)
     return
 
 
@@ -245,7 +245,7 @@ def do_extract(self, received=0):
 
     def update_url(self, received):
         args = self.common_args.copy()
-        play_type = 'empty' if received else 'play'
+        play_type = 'seek' if received else 'play'
         received = received if received else -1
         args['ls'] = play_type
         args['start'] = received + 1

From 46a83efd3d1f5327bbfd5049f32b4e429b5abc3f Mon Sep 17 00:00:00 2001
From: David Zhuang <cnbeining@gmail.com>
Date: Fri, 18 Aug 2017 00:49:10 -0400
Subject: [PATCH 0471/1225] [Fantasy] Add support

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/fantasy.py  | 52 ++++++++++++++++++++++++++++++
 3 files changed, 54 insertions(+)
 create mode 100644 src/you_get/extractors/fantasy.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 52dbb74988..eb79d250bc 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -19,6 +19,7 @@
     'douyu'            : 'douyutv',
     'ehow'             : 'ehow',
     'facebook'         : 'facebook',
+    'fantasy'          : 'fantasy',
     'fc2'              : 'fc2video',
     'flickr'           : 'flickr',
     'freesound'        : 'freesound',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 6d6f56cf94..ab5bc7c1bf 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -17,6 +17,7 @@
 from .douyutv import *
 from .ehow import *
 from .facebook import *
+from .fantasy import *
 from .fc2video import *
 from .flickr import *
 from .freesound import *
diff --git a/src/you_get/extractors/fantasy.py b/src/you_get/extractors/fantasy.py
new file mode 100644
index 0000000000..d072af06c0
--- /dev/null
+++ b/src/you_get/extractors/fantasy.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+__all__ = ['fantasy_download']
+
+from ..common import *
+import json
+import random
+from urllib.parse import urlparse, parse_qs
+
+
+def fantasy_download_by_id_channelId(id = 0, channelId = 0, output_dir = '.', merge = True, info_only = False,
+                                     **kwargs):
+    api_url = 'http://www.fantasy.tv/tv/playDetails.action?' \
+              'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format(id = id,
+                                                                         channelId = channelId,
+                                                                         t = str(random.random())
+                                                                         )
+    html = get_content(api_url)
+    html = json.loads(html)
+
+    if int(html['status']) != 100000:
+        raise Exception('API error!')
+
+    title = html['data']['tv']['title']
+
+    video_url = html['data']['tv']['videoPath']
+    type, ext, size = url_info(video_url)
+
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([video_url], title, ext, size, output_dir, merge = merge, headers = fake_headers)
+
+
+def fantasy_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    if 'fantasy.tv' not in url:
+        raise Exception('Wrong place!')
+
+    q = parse_qs(urlparse(url).query)
+
+    if 'tvId' not in q or 'channelId' not in q:
+        raise Exception('No enough arguments!')
+
+    tvId = q['tvId'][0]
+    channelId = q['channelId'][0]
+
+    fantasy_download_by_id_channelId(id = tvId, channelId = channelId, output_dir = output_dir, merge = merge,
+                                     info_only = info_only, **kwargs)
+
+
+site_info = "fantasy.tv"
+download = fantasy_download
+download_playlist = playlist_not_supported('fantasy.tv')

From 593610406bd029846758e5257660ad7938808ffa Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 21 Aug 2017 21:58:30 +0200
Subject: [PATCH 0472/1225] [baidu] tieba album API: set pe=1000

---
 src/you_get/extractors/baidu.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index d5efaf0bde..e264f68e84 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -134,8 +134,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             # handle albums
             kw = r1(r'kw=([^&]+)', html) or r1(r"kw:'([^']+)'", html)
             tid = r1(r'tid=(\d+)', html) or r1(r"tid:'([^']+)'", html)
-            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s' % (
-                kw, tid)
+            album_url = 'http://tieba.baidu.com/photo/g/bw/picture/list?kw=%s&tid=%s&pe=%s' % (kw, tid, 1000)
             album_info = json.loads(get_content(album_url))
             for i in album_info['data']['pic_list']:
                 urls.append(

From ca5e07cfa12115592b961d5a4b5760d1eb44e7de Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 23 Aug 2017 13:57:16 +0800
Subject: [PATCH 0473/1225] [youku]user-agent from youku ipad client has been
 blocked, switch to a normal desktop browser UA

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 863573d742..286c88ba66 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -38,7 +38,7 @@ def quote_cna(val):
 
 class Youku(VideoExtractor):
     name = "优酷 (Youku)"
-    mobile_ua = 'Mozilla/5.0 (iPad; CPU OS 10_1_1 like Mac OS X) AppleWebKit/602.2.14 (KHTML, like Gecko) Mobile/14B100'
+    mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
     dispatcher_url = 'vali.cp31.ott.cibntv.net'
 
     # Last updated: 2015-11-24

From 19d455de6b19d283eefe6bc3be8d4c9753a80279 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 23 Aug 2017 14:11:55 +0800
Subject: [PATCH 0474/1225] [iqiyi]get tvid and vid from js inlined in html;
 fix #2294

---
 src/you_get/extractors/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 12508fde86..8e93959acd 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -131,10 +131,10 @@ def prepare(self, **kwargs):
             html = get_html(self.url)
             tvid = r1(r'#curid=(.+)_', self.url) or \
                    r1(r'tvid=([^&]+)', self.url) or \
-                   r1(r'data-player-tvid="([^"]+)"', html) or r1(r'tv(?:i|I)d=(.+?)\&', html)
+                   r1(r'data-player-tvid="([^"]+)"', html) or r1(r'tv(?:i|I)d=(.+?)\&', html) or r1(r'param\[\'tvid\'\]\s*=\s*"(.+?)"', html)
             videoid = r1(r'#curid=.+_(.*)$', self.url) or \
                       r1(r'vid=([^&]+)', self.url) or \
-                      r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(.+?)\&', html)
+                      r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(.+?)\&', html) or r1(r'param\[\'vid\'\]\s*=\s*"(.+?)"', html)
             self.vid = (tvid, videoid)
             info_u = 'http://mixer.video.iqiyi.com/jp/mixin/videos/' + tvid
             mixin = get_content(info_u)

From 0d89c45c491a2056e0e21f0b568855a1005f43a2 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 24 Aug 2017 17:06:23 +0800
Subject: [PATCH 0475/1225] [youtube]fix ytb short url patterns

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 986906d6ec..318353625d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -83,7 +83,7 @@ def get_url_from_vid(vid):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        return match1(url, r'youtu\.be/([^/]+)') or \
+        return match1(url, r'youtu\.be/([^?/]+)') or \
           match1(url, r'youtube\.com/embed/([^/?]+)') or \
           match1(url, r'youtube\.com/v/([^/?]+)') or \
           match1(url, r'youtube\.com/watch/([^/?]+)') or \

From aaefb0e2d06f73098057a78f2a52fce3c869a60b Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 25 Aug 2017 20:14:01 +0800
Subject: [PATCH 0476/1225] [bilibili]fix quality mapping for paid bangumi;
 remove debug print

---
 src/you_get/extractors/bilibili.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 4b801e6219..33c4544d86 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -41,11 +41,11 @@ class Bilibili(VideoExtractor):
     @staticmethod
     def bilibili_stream_type(urls):
         url = urls[0]
-        if 'hd.flv?' in url:
+        if 'hd.flv?' in url or '-112.flv' in url:
             return 'hdflv', 'flv'
         if '.flv?' in url:
             return 'flv', 'flv'
-        if 'hd.mp4?' in url:
+        if 'hd.mp4?' in url or '-48.mp4' in url:
             return 'hdmp4', 'mp4'
         if '.mp4?' in url:
             return 'mp4', 'mp4'
@@ -226,7 +226,6 @@ def bangumi_entry(self, **kwargs):
         cid = ep_info['danmaku']
 
         self.title = '{} [{} {}]'.format(self.title, index_title, long_title)
-        print(self.title)
         self.download_by_vid(cid, bangumi=True, **kwargs)
 
 

From a129903da61930472d1bb46a64a0e557cf4184b7 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 26 Aug 2017 04:37:06 +0800
Subject: [PATCH 0477/1225] [bilibili]support paid bangumi_movie; fix #2304

---
 src/you_get/extractors/bilibili.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 33c4544d86..593493cb25 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -51,14 +51,15 @@ def bilibili_stream_type(urls):
             return 'mp4', 'mp4'
         raise Exception('Unknown stream type')
 
-    def api_req(self, cid, quality, bangumi):
+    def api_req(self, cid, quality, bangumi, bangumi_movie=False, **kwargs):
         ts = str(int(time.time()))
         if not bangumi:
             params_str = 'cid={}&player=1&quality={}&ts={}'.format(cid, quality, ts)
             chksum = hashlib.md5(bytes(params_str+self.SEC1, 'utf8')).hexdigest()
             api_url = self.api_url + params_str + '&sign=' + chksum
         else:
-            params_str = 'cid={}&module=bangumi&player=1&quality={}&ts={}'.format(cid, quality, ts)
+            mod = 'movie' if bangumi_movie else 'bangumi'
+            params_str = 'cid={}&module={}&player=1&quality={}&ts={}'.format(cid, mod, quality, ts)
             chksum = hashlib.md5(bytes(params_str+self.SEC2, 'utf8')).hexdigest()
             api_url = self.bangumi_api_url + params_str + '&sign=' + chksum
 
@@ -94,12 +95,12 @@ def download_by_vid(self, cid, bangumi, **kwargs):
         if not info_only or stream_id:
 # won't be None
             qlt = self.fmt2qlt.get(quality)
-            api_xml = self.api_req(cid, qlt, bangumi)
+            api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
             self.parse_bili_xml(api_xml)
             self.danmuku = get_danmuku_xml(cid)
         else:
             for qlt in range(4, 0, -1):
-                api_xml = self.api_req(cid, qlt, bangumi)
+                api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
                 self.parse_bili_xml(api_xml)
 
     def prepare(self, **kwargs):
@@ -137,9 +138,9 @@ def movie_entry(self, **kwargs):
         patt = r"var\s*aid\s*=\s*'(\d+)'"
         aid = re.search(patt, self.page).group(1)
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
+# better ideas for bangumi_movie titles?
         self.title = page_list[0]['pagename']
-# False for is_bangumi, old interface works for all free items
-        self.download_by_vid(page_list[0]['cid'], False, **kwargs)
+        self.download_by_vid(page_list[0]['cid'], True, bangumi_movie=True, **kwargs)
 
     def entry(self, **kwargs):
 # tencent player

From c036eae22aa3dda74b1a52f123a2955bab883814 Mon Sep 17 00:00:00 2001
From: Rafael Slonik <rslonik@gmail.com>
Date: Fri, 25 Aug 2017 19:03:40 -0300
Subject: [PATCH 0478/1225] Facebook extrator won't detect title for some
 videos and you-get breaks; so set title = url and we're good to go

---
 src/you_get/extractors/facebook.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 9eb9fae953..7134c56cb6 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -9,6 +9,10 @@ def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs
     html = get_html(url)
 
     title = r1(r'<title id="pageTitle">(.+)</title>', html)
+
+    if title is None:
+      title = url
+
     sd_urls = list(set([
         unicodize(str.replace(i, '\\/', '/'))
         for i in re.findall(r'sd_src_no_ratelimit:"([^"]*)"', html)

From 8a68cf8bd9b6d66acaf59e8097bde19e7eb8945d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Aug 2017 04:05:56 +0200
Subject: [PATCH 0479/1225] [iwara] fix info_only

---
 src/you_get/extractors/iwara.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 21b44608d8..50d14fb8d8 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -28,8 +28,9 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     down_urls=data[0]['uri']
     print_info(down_urls,title+data[0]['resolution'],type,size)
 
-    download_urls([down_urls], title, ext, size, output_dir, merge = merge,headers=headers)
+    if not info_only:
+        download_urls([down_urls], title, ext, size, output_dir, merge = merge,headers=headers)
 
 site_info = "iwara"
 download = iwara_download
-download_playlist = playlist_not_supported('iwara')
\ No newline at end of file
+download_playlist = playlist_not_supported('iwara')

From f92c21b16e413fe5d4acd756e18234e9c63c9618 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 29 Aug 2017 14:15:53 +0800
Subject: [PATCH 0480/1225] [ku6]new url pattern

---
 src/you_get/extractors/ku6.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index 9996d7e4d1..8ff3dd7e13 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -36,6 +36,31 @@ def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwarg
                 r'http://v.ku6.com/show/(.*)\.\.\.html',
                 r'http://my.ku6.com/watch\?.*v=(.*)\.\..*']
         id = r1_of(patterns, url)
+
+    if id is None:
+        # http://www.ku6.com/2017/detail-zt.html?vid=xvqTmvZrH8MNvErpvRxFn3
+        page = get_content(url)
+        meta = re.search(r'detailDataMap=(\{.+?\});', page)
+        if meta is not None:
+            meta = meta.group(1)
+        else:
+            raise Exception('Unsupported url')
+        vid = re.search(r'vid=([^&]+)', url)
+        if vid is not None:
+            vid = vid.group(1)
+        else:
+            raise Exception('Unsupported url')
+        this_meta = re.search(vid+':\{(.+?)\}', meta)
+        if this_meta is not None:
+            this_meta = this_meta.group(1)
+            title = re.search('title:"(.+?)"', this_meta).group(1)
+            video_url = re.search('playUrl:"(.+?)"', this_meta).group(1)
+        video_size = url_size(video_url)
+        print_info(site_info, title, 'mp4', video_size)
+        if not info_only:
+            download_urls([video_url], title, 'mp4', video_size, output_dir, merge=merge, **kwargs)
+        return
+
     ku6_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
 
 def baidu_ku6(url):

From 083bc291d2cbc7e36e46f04ed61bbb52d9e9dc68 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 29 Aug 2017 15:38:23 +0800
Subject: [PATCH 0481/1225] [ku6]some vids are quoted

---
 src/you_get/extractors/ku6.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index 8ff3dd7e13..c827eafd81 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -50,7 +50,7 @@ def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwarg
             vid = vid.group(1)
         else:
             raise Exception('Unsupported url')
-        this_meta = re.search(vid+':\{(.+?)\}', meta)
+        this_meta = re.search('"?'+vid+'"?:\{(.+?)\}', meta)
         if this_meta is not None:
             this_meta = this_meta.group(1)
             title = re.search('title:"(.+?)"', this_meta).group(1)

From 9248e6f8969e52cc8daa532cef6c25d71c095eca Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 30 Aug 2017 19:58:42 +0800
Subject: [PATCH 0482/1225] update help message

---
 src/you_get/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f613427c64..1446c960bd 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1102,6 +1102,7 @@ def version():
     -d | --debug                        Show traceback and other debug info.
     -I | --input-file                   Read non-playlist urls from file.
     -P | --password <PASSWORD>          Set video visit password to PASSWORD.
+    -l | --playlist                     Download a playlist.
     '''
 
     short_opts = 'Vhfiuc:ndF:O:o:p:x:y:s:t:I:P:'

From e74fc8ee8bf0acbdb477ece7113886e58140440f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Sep 2017 22:54:41 +0200
Subject: [PATCH 0483/1225] [google+] fix post_date

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index c4e1a3f21a..9e363afade 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -61,7 +61,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         if not real_urls:
             real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
             real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
-        post_date = r1(r'"(20\d\d-[01]\d-[0123]\d)"', html)
+        post_date = r1(r'"?(20\d\d-[01]\d-[0123]\d)"?', html)
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id
 

From 0832bcc4e494135df7b3329b5cf7bfe75836a407 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 2 Sep 2017 03:45:54 +0200
Subject: [PATCH 0484/1225] [google+] fix more

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 9e363afade..9321ac5014 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -61,7 +61,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         if not real_urls:
             real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
             real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
-        post_date = r1(r'"?(20\d\d-[01]\d-[0123]\d)"?', html)
+        post_date = r1(r'"?(20\d\d[-/][01]\d[-/][0123]\d)"?', html)
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id
 

From 2c32db485e5c33876483e3164e5911be470d1c13 Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Sun, 3 Sep 2017 16:49:14 -0700
Subject: [PATCH 0485/1225] [common] retry if response.read() timeout

---
 src/you_get/common.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1446c960bd..6e58fb81c5 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -597,13 +597,18 @@ def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers
 
         with open(temp_filepath, open_mode) as output:
             while True:
-                buffer = response.read(1024 * 256)
+                buffer = None
+                try:
+                    buffer = response.read(1024 * 256)
+                except socket.timeout:
+                    pass
                 if not buffer:
                     if received == file_size: # Download finished
                         break
-                    else: # Unexpected termination. Retry request
-                        tmp_headers['Range'] = 'bytes=' + str(received) + '-'
-                        response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
+                    # Unexpected termination. Retry request
+                    tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+                    response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
+                    continue
                 output.write(buffer)
                 received += len(buffer)
                 if bar:

From 22a602cdab9d3687e368463d111850ced3d50e82 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 5 Sep 2017 02:20:44 +0800
Subject: [PATCH 0486/1225] [funshion]rewrite

---
 src/you_get/extractors/funshion.py | 355 +++++++++++++++++------------
 1 file changed, 214 insertions(+), 141 deletions(-)

diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
index 6a24b46402..1f7156b5e3 100644
--- a/src/you_get/extractors/funshion.py
+++ b/src/you_get/extractors/funshion.py
@@ -1,150 +1,223 @@
 #!/usr/bin/env python
 
+import json
+import urllib.parse
+import base64
+import binascii
+import re
+
+from ..extractors import VideoExtractor
+from ..util import log
+from ..common import get_content, playlist_not_supported
+
 __all__ = ['funshion_download']
 
-from ..common import *
-import urllib.error
-import json
 
-#----------------------------------------------------------------------
-def funshion_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    """"""
-    if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):  #single video
-        funshion_download_by_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif re.match(r'http://www.fun.tv/vplay/.*g-(\w+)', url):  #whole drama
-        funshion_download_by_drama_url(url, output_dir=output_dir, merge=merge, info_only=info_only)
-    else:
-        return
-
-# Logics for single video until drama
-#----------------------------------------------------------------------
-def funshion_download_by_url(url, output_dir = '.', merge = False, info_only = False):
-    """lots of stuff->None
-    Main wrapper for single video download.
-    """
+class KBaseMapping:
+    def __init__(self, base=62):
+        self.base = base
+        mapping_table = [str(num) for num in range(10)]
+        for i in range(26):
+            mapping_table.append(chr(i + ord('a')))
+        for i in range(26):
+            mapping_table.append(chr(i + ord('A')))
+
+        self.mapping_table = mapping_table[:self.base]
+
+    def mapping(self, num):
+        res = []
+        while num > 0:
+            res.append(self.mapping_table[num % self.base])
+            num = num // self.base
+        return ''.join(res[::-1])
+
+
+class Funshion(VideoExtractor):
+    name = "funshion"
+    stream_types = [
+        {'id': 'sdvd'},
+        {'id': 'sdvd_h265'},
+        {'id': 'hd'},
+        {'id': 'hd_h265'},
+        {'id': 'dvd'},
+        {'id': 'dvd_h265'},
+        {'id': 'tv'},
+        {'id': 'tv_h265'}
+    ]
+    a_mobile_url = 'http://m.fun.tv/implay/?mid=302555'
+    video_ep = 'http://pv.funshion.com/v7/video/play/?id={}&cl=mweb&uc=111'
+    media_ep = 'http://pm.funshion.com/v7/media/play/?id={}&cl=mweb&uc=111'
+    coeff = None
+
+    @classmethod
+    def fetch_magic(cls, url):
+        def search_dict(a_dict, target):
+            for key, val in a_dict.items():
+                if val == target:
+                    return key
+
+        magic_list = []
+
+        page = get_content(url)
+        src = re.findall(r'src="(.+?)"', page)
+        js = [path for path in src if path.endswith('.js')]
+
+        host = 'http://' + urllib.parse.urlparse(url).netloc
+        js_path = [urllib.parse.urljoin(host, rel_path) for rel_path in js]
+
+        for p in js_path:
+            if 'mtool' in p or 'mcore' in p:
+                js_text = get_content(p)
+                hit = re.search(r'\(\'(.+?)\',(\d+),(\d+),\'(.+?)\'\.split\(\'\|\'\),\d+,\{\}\)', js_text)
+
+                code = hit.group(1)
+                base = hit.group(2)
+                size = hit.group(3)
+                names = hit.group(4).split('|')
+
+                mapping = KBaseMapping(base=int(base))
+                sym_to_name = {}
+                for no in range(int(size), 0, -1):
+                    no_in_base = mapping.mapping(no)
+                    val = names[no] if no < len(names) and names[no] else no_in_base
+                    sym_to_name[no_in_base] = val
+
+                moz_ec_name = search_dict(sym_to_name, 'mozEcName')
+                push = search_dict(sym_to_name, 'push')
+                patt = '{}\.{}\("(.+?)"\)'.format(moz_ec_name, push)
+                ec_list = re.findall(patt, code)
+                [magic_list.append(sym_to_name[ec]) for ec in ec_list]
+        return magic_list
+
+    @classmethod
+    def get_coeff(cls, magic_list):
+        magic_set = set(magic_list)
+        no_dup = []
+        for item in magic_list:
+            if item in magic_set:
+                magic_set.remove(item)
+                no_dup.append(item)
+        # really necessary?
+
+        coeff = [0, 0, 0, 0]
+        for num_pair in no_dup:
+            idx = int(num_pair[-1])
+            val = int(num_pair[:-1], 16)
+            coeff[idx] = val
+
+        return coeff
+
+    @classmethod
+    def funshion_decrypt(cls, a_bytes, coeff):
+        res_list = []
+        pos = 0
+        while pos < len(a_bytes):
+            a = a_bytes[pos]
+            if pos == len(a_bytes) - 1:
+                res_list.append(a)
+                pos += 1
+            else:
+                b = a_bytes[pos + 1]
+                m = a * coeff[0] + b * coeff[2]
+                n = a * coeff[1] + b * coeff[3]
+                res_list.append(m & 0xff)
+                res_list.append(n & 0xff)
+                pos += 2
+        return bytes(res_list).decode('utf8')
+
+    @classmethod
+    def funshion_decrypt_str(cls, a_str, coeff):
+        # r'.{27}0' pattern, untested
+        if len(a_str) == 28 and a_str[-1] == '0':
+            data_bytes = base64.b64decode(a_str[:27] + '=')
+            clear = cls.funshion_decrypt(data_bytes, coeff)
+            return binascii.hexlify(clear.encode('utf8')).upper()
+
+        data_bytes = base64.b64decode(a_str[2:])
+        return cls.funshion_decrypt(data_bytes, coeff)
+
+    @classmethod
+    def checksum(cls, sha1_str):
+        if len(sha1_str) != 41:
+            return False
+        if not re.match(r'[0-9A-Za-z]{41}', sha1_str):
+            return False
+        sha1 = sha1_str[:-1]
+        if (15 & sum([int(char, 16) for char in sha1])) == int(sha1_str[-1], 16):
+            return True
+        return False
+
+    @classmethod
+    def get_cdninfo(cls, hashid):
+        url = 'http://jobsfe.funshion.com/query/v1/mp4/{}.json'.format(hashid)
+        meta = json.loads(get_content(url, decoded=False).decode('utf8'))
+        return meta['playlist'][0]['urls']
+
+    @classmethod
+    def dec_playinfo(cls, info, coeff):
+        res = None
+        clear = cls.funshion_decrypt_str(info['infohash'], coeff)
+        if cls.checksum(clear):
+            res = dict(hashid=clear[:40], token=cls.funshion_decrypt_str(info['token'], coeff))
+        else:
+            clear = cls.funshion_decrypt_str(info['infohash_prev'], coeff)
+            if cls.checksum(clear):
+                res = dict(hashid=clear[:40], token=cls.funshion_decrypt_str(info['token_prev'], coeff))
+        return res
+
+    def prepare(self, **kwargs):
+        if self.__class__.coeff is None:
+            magic_list = self.__class__.fetch_magic(self.__class__.a_mobile_url)
+            self.__class__.coeff = self.__class__.get_coeff(magic_list)
+
+        if 'title' not in kwargs:
+            url = 'http://pv.funshion.com/v5/video/profile/?id={}&cl=mweb&uc=111'.format(self.vid)
+            meta = json.loads(get_content(url))
+            self.title = meta['name']
+        else:
+            self.title = kwargs['title']
+
+        ep_url = self.__class__.video_ep if 'single_video' in kwargs else self.__class__.media_ep
+
+        url = ep_url.format(self.vid)
+        meta = json.loads(get_content(url))
+        streams = meta['playlist']
+        for stream in streams:
+            definition = stream['code']
+            for s in stream['playinfo']:
+                codec = 'h' + s['codec'][2:]
+                # h.264 -> h264
+                for st in self.__class__.stream_types:
+                    s_id = '{}_{}'.format(definition, codec)
+                    if codec == 'h264':
+                        s_id = definition
+                    if s_id == st['id']:
+                        clear_info = self.__class__.dec_playinfo(s, self.__class__.coeff)
+                        cdn_list = self.__class__.get_cdninfo(clear_info['hashid'])
+                        base_url = cdn_list[0]
+                        vf = urllib.parse.quote(s['vf'])
+                        video_size = int(s['filesize'])
+                        token = urllib.parse.quote(base64.b64encode(clear_info['token'].encode('utf8')))
+                        video_url = '{}?token={}&vf={}'.format(base_url, token, vf)
+                        self.streams[s_id] = dict(size=video_size, src=[video_url], container='mp4')
+
+
+def funshion_download(url, **kwargs):
     if re.match(r'http://www.fun.tv/vplay/v-(\w+)', url):
-        match = re.search(r'http://www.fun.tv/vplay/v-(\d+)(.?)', url)
-    vid = match.group(1)
-    funshion_download_by_vid(vid, output_dir=output_dir, merge=merge, info_only=info_only)
-
-#----------------------------------------------------------------------
-def funshion_download_by_vid(vid, output_dir = '.', merge = False, info_only = False):
-    """vid->None
-    Secondary wrapper for single video download.
-    """
-    title = funshion_get_title_by_vid(vid)
-    url_list = funshion_vid_to_urls(vid)
-
-    for url in url_list:
-        type, ext, size = url_info(url)
-        print_info(site_info, title, type, size)
-
-    if not info_only:
-        download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
-
-#----------------------------------------------------------------------
-def funshion_get_title_by_vid(vid):
-    """vid->str
-    Single video vid to title."""
-    html = get_content('http://pv.funshion.com/v5/video/profile?id={vid}&cl=aphone&uc=5'.format(vid = vid))
-    c = json.loads(html)
-    return c['name']
-
-#----------------------------------------------------------------------
-def funshion_vid_to_urls(vid):
-    """str->str
-    Select one resolution for single video download."""
-    html = get_content('http://pv.funshion.com/v5/video/play/?id={vid}&cl=aphone&uc=5'.format(vid = vid))
-    return select_url_from_video_api(html)
-
-#Logics for drama until helper functions
-#----------------------------------------------------------------------
-def funshion_download_by_drama_url(url, output_dir = '.', merge = False, info_only = False):
-    """str->None
-    url = 'http://www.fun.tv/vplay/g-95785/'
-    """
-    id = r1(r'http://www.fun.tv/vplay/.*g-(\d+)', url)
-    video_list = funshion_drama_id_to_vid(id)
-
-    for video in video_list:
-        funshion_download_by_id((video[0], id), output_dir=output_dir, merge=merge, info_only=info_only)
-        # id is for drama, vid not the same as the ones used in single video
-
-#----------------------------------------------------------------------
-def funshion_download_by_id(vid_id_tuple, output_dir = '.', merge = False, info_only = False):
-    """single_episode_id, drama_id->None
-    Secondary wrapper for single drama video download.
-    """
-    (vid, id) = vid_id_tuple
-    title = funshion_get_title_by_id(vid, id)
-    url_list = funshion_id_to_urls(vid)
-
-    for url in url_list:
-        type, ext, size = url_info(url)
-        print_info(site_info, title, type, size)
-
-    if not info_only:
-        download_urls(url_list, title, ext, total_size=None, output_dir=output_dir, merge=merge)
-
-#----------------------------------------------------------------------
-def funshion_drama_id_to_vid(episode_id):
-    """int->[(int,int),...]
-    id: 95785
-    ->[('626464', '1'), ('626466', '2'), ('626468', '3'),...
-    Drama ID to vids used in drama.
-
-    **THIS VID IS NOT THE SAME WITH THE ONES USED IN SINGLE VIDEO!!**
-    """
-    html = get_content('http://pm.funshion.com/v5/media/episode?id={episode_id}&cl=aphone&uc=5'.format(episode_id = episode_id))
-    c = json.loads(html)
-    #{'definition': [{'name': '流畅', 'code': 'tv'}, {'name': '标清', 'code': 'dvd'}, {'name': '高清', 'code': 'hd'}], 'retmsg': 'ok', 'total': '32', 'sort': '1', 'prevues': [], 'retcode': '200', 'cid': '2', 'template': 'grid', 'episodes': [{'num': '1', 'id': '624728', 'still': None, 'name': '第1集', 'duration': '45:55'}, ], 'name': '太行山上', 'share': 'http://pm.funshion.com/v5/media/share?id=201554&num=', 'media': '201554'}
-    return [(i['id'], i['num']) for i in c['episodes']]
-
-#----------------------------------------------------------------------
-def funshion_id_to_urls(id):
-    """int->list of URL
-    Select video URL for single drama video.
-    """
-    html = get_content('http://pm.funshion.com/v5/media/play/?id={id}&cl=aphone&uc=5'.format(id = id))
-    return select_url_from_video_api(html)
-
-#----------------------------------------------------------------------
-def funshion_get_title_by_id(single_episode_id, drama_id):
-    """single_episode_id, drama_id->str
-    This is for full drama.
-    Get title for single drama video."""
-    html = get_content('http://pm.funshion.com/v5/media/episode?id={id}&cl=aphone&uc=5'.format(id = drama_id))
-    c = json.loads(html)
-
-    for i in c['episodes']:
-        if i['id'] == str(single_episode_id):
-            return c['name'] + ' - ' + i['name']
-
-# Helper functions.
-#----------------------------------------------------------------------
-def select_url_from_video_api(html):
-    """str(html)->str(url)
-
-    Choose the best one.
-
-    Used in both single and drama download.
-
-    code definition:
-    {'tv': 'liuchang',
-    'dvd': 'biaoqing',
-    'hd': 'gaoqing',
-    'sdvd': 'chaoqing'}"""
-    c = json.loads(html)
-    #{'retmsg': 'ok', 'retcode': '200', 'selected': 'tv', 'mp4': [{'filename': '', 'http': 'http://jobsfe.funshion.com/query/v1/mp4/7FCD71C58EBD4336DF99787A63045A8F3016EC51.json', 'filesize': '96748671', 'code': 'tv', 'name': '流畅', 'infohash': '7FCD71C58EBD4336DF99787A63045A8F3016EC51'}...], 'episode': '626464'}
-    video_dic = {}
-    for i in c['mp4']:
-        video_dic[i['code']] = i['http']
-    quality_preference_list = ['sdvd', 'hd', 'dvd', 'sd']
-    url = [video_dic[quality] for quality in quality_preference_list if quality in video_dic][0]
-    html = get_html(url)
-    c = json.loads(html)
-    #'{"return":"succ","client":{"ip":"107.191.**.**","sp":"0","loc":"0"},"playlist":[{"bits":"1638400","tname":"dvd","size":"555811243","urls":["http:\\/\\/61.155.217.4:80\\/play\\/1E070CE31DAA1373B667FD23AA5397C192CA6F7F.mp4",...]}]}'
-    return [i['urls'][0] for i in c['playlist']]
+        vid = re.search(r'http://www.fun.tv/vplay/v-(\w+)', url).group(1)
+        Funshion().download_by_vid(vid, single_video=True, **kwargs)
+    elif re.match(r'http://www.fun.tv/vplay/.*g-(\w+)', url):
+        epid = re.search(r'http://www.fun.tv/vplay/.*g-(\w+)', url).group(1)
+        url = 'http://pm.funshion.com/v5/media/episode?id={}&cl=mweb&uc=111'.format(epid)
+        meta = json.loads(get_content(url))
+        drama_name = meta['name']
+
+        extractor = Funshion()
+        for ep in meta['episodes']:
+            title = '{}_{}_{}'.format(drama_name, ep['num'], ep['name'])
+            extractor.download_by_vid(ep['id'], title=title, **kwargs)
+    else:
+        log.wtf('Unknown url pattern')
 
 site_info = "funshion"
 download = funshion_download

From 0a00ff3c463e1f273862c1fba8bd85b729f6dd52 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 5 Sep 2017 12:36:50 +0800
Subject: [PATCH 0487/1225] [embed]support bokecc cloud

---
 src/you_get/extractors/embed.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 8b12a07cd4..eb758431eb 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -15,6 +15,7 @@
 from .yinyuetai import yinyuetai_download_by_id
 from .youku import youku_download_by_vid
 from . import iqiyi
+from . import bokecc
 
 """
 refer to http://open.youku.com/tools
@@ -58,6 +59,8 @@
 iqiyi_patterns = [r'(?:\"|\')(https?://dispatcher\.video\.qiyi\.com\/disp\/shareplayer\.swf\?.+?)(?:\"|\')',
                   r'(?:\"|\')(https?://open\.iqiyi\.com\/developer\/player_js\/coopPlayerIndex\.html\?.+?)(?:\"|\')']
 
+bokecc_patterns = [r'bokecc\.com/flash/pocle/player\.swf\?siteid=(.+?)&vid=(.{32})']
+
 recur_limit = 3
 
 
@@ -107,6 +110,11 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         iqiyi.download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
+    bokecc_metas = matchall(content, bokecc_patterns)
+    for meta in bokecc_metas:
+        found = True
+        bokecc.bokecc_download_by_id(meta[1], output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
     if found:
         return True
 

From ca490abead2b4ac03d3c44b06477f948e2139a58 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 5 Sep 2017 12:37:50 +0800
Subject: [PATCH 0488/1225] [bokecc]fix a unbound error; set title to vid if
 cannot fetch it from api data

---
 src/you_get/extractors/bokecc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bokecc.py b/src/you_get/extractors/bokecc.py
index 8566e8282f..542e83ba30 100644
--- a/src/you_get/extractors/bokecc.py
+++ b/src/you_get/extractors/bokecc.py
@@ -52,10 +52,13 @@ def prepare(self, vid = '', title = None, **kwargs):
             raise 
 
         if title is None:
-            self.title = '_'.join([i.text for i in tree.iterfind('video/videomarks/videomark/markdesc')])
+            self.title = '_'.join([i.text for i in self.tree.iterfind('video/videomarks/videomark/markdesc')])
         else:
             self.title = title
 
+        if not title:
+            self.title = vid
+
         for i in self.tree.iterfind('video/quality'):
             quality = i.attrib ['value']
             url = i[0].attrib['playurl']

From c95146a4e279a8dcd08e50b9af1775776613ebdf Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 5 Sep 2017 13:41:52 +0800
Subject: [PATCH 0489/1225] [qingting]support qingting; close #1243

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/qingting.py | 50 ++++++++++++++++++++++++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 src/you_get/extractors/qingting.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1446c960bd..98574bb24a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -62,6 +62,7 @@
     'pinterest'        : 'pinterest',
     'pixnet'           : 'pixnet',
     'pptv'             : 'pptv',
+    'qingting'         : 'qingting',
     'qq'               : 'qq',
     'quanmin'          : 'quanmin',
     'showroom-live'    : 'showroom',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index ab5bc7c1bf..b078db0876 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -56,6 +56,7 @@
 from .pixnet import *
 from .pptv import *
 from .qie import *
+from .qingting import *
 from .qq import *
 from .showroom import *
 from .sina import *
diff --git a/src/you_get/extractors/qingting.py b/src/you_get/extractors/qingting.py
new file mode 100644
index 0000000000..9859d4be95
--- /dev/null
+++ b/src/you_get/extractors/qingting.py
@@ -0,0 +1,50 @@
+import json
+import re
+
+from ..common import get_content, playlist_not_supported, url_size
+from ..extractors import VideoExtractor
+from ..util import log
+
+__all__ = ['qingting_download_by_url']
+
+
+class Qingting(VideoExtractor):
+    # every resource is described by its channel id and program id
+    # so vid is tuple (chaanel_id, program_id)
+
+    name = 'Qingting'
+    stream_types = [
+        {'id': '_default'}
+    ]
+
+    ep = 'http://i.qingting.fm/wapi/channels/{}/programs/{}'
+    file_host = 'http://od.qingting.fm/{}'
+    mobile_pt = r'channels\/(\d+)\/programs/(\d+)'
+
+    def prepare(self, **kwargs):
+        if self.vid is None:
+            hit = re.search(self.__class__.mobile_pt, self.url)
+            self.vid = (hit.group(1), hit.group(2))
+
+        ep_url = self.__class__.ep.format(self.vid[0], self.vid[1])
+        meta = json.loads(get_content(ep_url))
+
+        if meta['code'] != 0:
+            log.wtf(meta['message']['errormsg'])
+
+        file_path = self.__class__.file_host.format(meta['data']['file_path'])
+        self.title = meta['data']['name']
+        duration = str(meta['data']['duration']) + 's'
+
+        self.streams['_default'] = {'src': [file_path], 'video_profile': duration, 'container': 'm4a'}
+
+    def extract(self, **kwargs):
+        self.streams['_default']['size'] = url_size(self.streams['_default']['src'][0])
+
+
+def qingting_download_by_url(url, **kwargs):
+    Qingting().download_by_url(url, **kwargs)
+
+site_info = 'Qingting'
+download = qingting_download_by_url
+download_playlist = playlist_not_supported('Qingting')

From 0fba6829d40b1ddcc151dc74ea43598928a9590f Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 6 Sep 2017 20:59:14 +0800
Subject: [PATCH 0490/1225] [youku]sleep 3s to handle server time out of sync

---
 src/you_get/extractors/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 286c88ba66..cb262bf960 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -148,6 +148,7 @@ def prepare(self, **kwargs):
             self.password = kwargs['password']
 
         self.utid = fetch_cna()
+        time.sleep(3)
         self.youku_ups()
 
         if self.api_data.get('stream') is None:

From 276e2db065aed8c7fe4fb8a04f24991b1b13dd0e Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Thu, 7 Sep 2017 14:59:52 +0800
Subject: [PATCH 0491/1225] [google]fix google doc; close #2324

---
 src/you_get/extractors/google.py | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index 9321ac5014..adc477e8fe 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -99,20 +99,34 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
     elif service in ['docs', 'drive'] : # Google Docs
 
-        html = get_html(url, faker=True)
+        html = get_content(url, headers=fake_headers)
 
         title = r1(r'"title":"([^"]*)"', html) or r1(r'<meta itemprop="name" content="([^"]*)"', html)
         if len(title.split('.')) > 1:
             title = ".".join(title.split('.')[:-1])
 
-        docid = r1(r'"docid":"([^"]*)"', html)
+        docid = r1('/file/d/([^/]+)', url)
 
         request.install_opener(request.build_opener(request.HTTPCookieProcessor()))
 
-        request.urlopen(request.Request("https://docs.google.com/uc?id=%s&export=download" % docid))
-        real_url ="https://docs.google.com/uc?export=download&confirm=no_antivirus&id=%s" % docid
-
-        type, ext, size = url_info(real_url)
+        real_url = "https://docs.google.com/uc?export=download&confirm=no_antivirus&id=%s" % docid
+        redirected_url = get_location(real_url)
+        if real_url != redirected_url:
+# tiny file - get real url here
+            type, ext, size = url_info(redirected_url)
+            real_url = redirected_url
+        else:
+# huge file - the real_url is a confirm page and real url is in it
+            confirm_page = get_content(real_url)
+            hrefs = re.findall(r'href="(.+?)"', confirm_page)
+            for u in hrefs:
+                if u.startswith('/uc?export=download'):
+                    rel = unescape_html(u)
+            confirm_url = 'https://docs.google.com' + rel
+            real_url = get_location(confirm_url)
+            _, ext, size = url_info(real_url, headers=fake_headers)
+            if size is None:
+                size = 0
 
         print_info(site_info, title, ext, size)
         if not info_only:

From 9c55f9395519f45c5768169fb7db9f4b79150b06 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Fri, 8 Sep 2017 21:05:16 +0800
Subject: [PATCH 0492/1225] [fantasy]request with referer

---
 src/you_get/extractors/fantasy.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/fantasy.py b/src/you_get/extractors/fantasy.py
index d072af06c0..3c7bee255c 100644
--- a/src/you_get/extractors/fantasy.py
+++ b/src/you_get/extractors/fantasy.py
@@ -24,11 +24,13 @@ def fantasy_download_by_id_channelId(id = 0, channelId = 0, output_dir = '.', me
     title = html['data']['tv']['title']
 
     video_url = html['data']['tv']['videoPath']
-    type, ext, size = url_info(video_url)
+    headers = fake_headers.copy()
+    headers['Referer'] = api_url
+    type, ext, size = url_info(video_url, headers=headers)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([video_url], title, ext, size, output_dir, merge = merge, headers = fake_headers)
+        download_urls([video_url], title, ext, size, output_dir, merge = merge, headers = headers)
 
 
 def fantasy_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):

From f916d837e8d027257be05e145257519686aa383c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sat, 9 Sep 2017 09:28:38 +0800
Subject: [PATCH 0493/1225] fix pan.baidu 1. change url pattern to support
 https protocol 2. move print_info and sleep to the entry function. old impl
 will block for 5 secs with info_only

---
 src/you_get/extractors/baidu.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index e264f68e84..6f558e31dc 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -104,9 +104,12 @@ def baidu_download_album(aid, output_dir='.', merge=True, info_only=False):
 
 def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=False, **kwargs):
 
-    if re.match(r'http://pan.baidu.com', url):
+    if re.match(r'https?://pan.baidu.com', url):
         real_url, title, ext, size = baidu_pan_download(url)
+        print_info('BaiduPan', title, ext, size)
         if not info_only:
+            print('Hold on...')
+            time.sleep(5)
             download_urls([real_url], title, ext, size,
                           output_dir, url, merge=merge, faker=True)
     elif re.match(r'http://music.baidu.com/album/\d+', url):
@@ -209,9 +212,6 @@ def baidu_pan_download(url):
     title_wrapped = json.loads('{"wrapper":"%s"}' % title)
     title = title_wrapped['wrapper']
     logging.debug(real_url)
-    print_info(site_info, title, ext, size)
-    print('Hold on...')
-    time.sleep(5)
     return real_url, title, ext, size
 
 

From 251857407b4b28b83fac19e0e6d74313840cb114 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Sun, 10 Sep 2017 12:52:03 +0800
Subject: [PATCH 0494/1225] [qie_video]add 1080p stream_types

---
 src/you_get/extractors/qie_video.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/qie_video.py b/src/you_get/extractors/qie_video.py
index c5d96e708b..9cf6ef1092 100644
--- a/src/you_get/extractors/qie_video.py
+++ b/src/you_get/extractors/qie_video.py
@@ -12,6 +12,7 @@ class QieVideo(VideoExtractor):
     cdn = 'http://qietv-play.wcs.8686c.com/'
     ep = 'http://api.qiecdn.com/api/v1/video/stream/{}'
     stream_types = [
+        {'id':'1080p', 'video_profile':'1920x1080', 'container':'m3u8'},
         {'id':'720p', 'video_profile':'1280x720', 'container':'m3u8'},
         {'id':'480p', 'video_profile':'853x480', 'container':'m3u8'}
     ]

From a534c11e947be993dd8e44a30cb83467376ade96 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 13:23:02 +0200
Subject: [PATCH 0495/1225] [embed] add dailymotion patterns

---
 src/you_get/extractors/embed.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index eb758431eb..8b1c45550f 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -5,6 +5,7 @@
 from ..common import *
 
 from .bilibili import bilibili_download
+from .dailymotion import dailymotion_download
 from .iqiyi import iqiyi_download_by_vid
 from .le import letvcloud_download_by_vu
 from .netease import netease_download
@@ -47,6 +48,8 @@
 
 vimeo_embed_patters = [ 'player\.vimeo\.com/video/(\d+)' ]
 
+dailymotion_embed_patterns = [ 'www\.dailymotion\.com/embed/video/(\w+)' ]
+
 """
 check the share button on http://www.bilibili.com/video/av5079467/
 """
@@ -99,6 +102,11 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
         found = True
         vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
 
+    urls = matchall(content, dailymotion_embed_patterns)
+    for url in urls:
+        found = True
+        dailymotion_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+
     aids = matchall(content, bilibili_embed_patterns)
     for aid in aids:
         found = True

From e3ebedc6adebc0adb602040fcc2d9749e7642b36 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 13:44:54 +0200
Subject: [PATCH 0496/1225] [dailymotion] WTF?

---
 src/you_get/extractors/dailymotion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index 682d8e4dc1..d0b8115df8 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -32,7 +32,7 @@ def dailymotion_download(url, info_only=False, **kwargs):
 
     print_info(site_info, title, mime, size)
     if not info_only:
-        download_urls([real_url], title, ext, size, **kwargs)
+        FUCK_OFF([real_url], title, ext, size, **kwargs)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From ea033aeaa800002de7ee2525bf4afaaeb0b2ddec Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 14:21:58 +0200
Subject: [PATCH 0497/1225] [dailymotion] do not use **kwargs in download_urls

---
 src/you_get/extractors/dailymotion.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/dailymotion.py b/src/you_get/extractors/dailymotion.py
index d0b8115df8..789dff453f 100644
--- a/src/you_get/extractors/dailymotion.py
+++ b/src/you_get/extractors/dailymotion.py
@@ -10,7 +10,7 @@ def rebuilt_url(url):
     aid = path.split('/')[-1].split('_')[0]
     return 'http://www.dailymotion.com/embed/video/{}?autoplay=1'.format(aid)
 
-def dailymotion_download(url, info_only=False, **kwargs):
+def dailymotion_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """Downloads Dailymotion videos by URL.
     """
 
@@ -32,7 +32,7 @@ def dailymotion_download(url, info_only=False, **kwargs):
 
     print_info(site_info, title, mime, size)
     if not info_only:
-        FUCK_OFF([real_url], title, ext, size, **kwargs)
+        download_urls([real_url], title, ext, size, output_dir=output_dir, merge=merge)
 
 site_info = "Dailymotion.com"
 download = dailymotion_download

From 24f89d3981276ee65d12ab53d43de88c3b0ce960 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 14:28:17 +0200
Subject: [PATCH 0498/1225] [embed] remove some 'title=title' -- these
 parameters are not even defined

---
 src/you_get/extractors/embed.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 8b1c45550f..3a36a3004c 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -95,17 +95,17 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
     urls = matchall(content, netease_embed_patterns)
     for url in urls:
         found = True
-        netease_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        netease_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
 
     urls = matchall(content, vimeo_embed_patters)
     for url in urls:
         found = True
-        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
+        vimeo_download_by_id(url, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
 
     urls = matchall(content, dailymotion_embed_patterns)
     for url in urls:
         found = True
-        dailymotion_download(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        dailymotion_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
 
     aids = matchall(content, bilibili_embed_patterns)
     for aid in aids:

From 21830e367f0e849010117b73efcc2c4c00d914ab Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 14:29:40 +0200
Subject: [PATCH 0499/1225] [embed] oops

---
 src/you_get/extractors/embed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 3a36a3004c..fa05181fd6 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -100,7 +100,7 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
     urls = matchall(content, vimeo_embed_patters)
     for url in urls:
         found = True
-        vimeo_download_by_id(url, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
+        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
 
     urls = matchall(content, dailymotion_embed_patterns)
     for url in urls:

From 936383d092e07d19852df5c2a03e1705c2f59047 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 15:32:48 +0200
Subject: [PATCH 0500/1225] version 0.4.900

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index b3de196f94..bc64e7a5ad 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.803'
+__version__ = '0.4.900'

From e8c7d9e6b2b6cf8ffa9fff5baee7f81c553b5f5c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 11 Sep 2017 19:03:08 +0200
Subject: [PATCH 0501/1225] [google+] fix more post_date

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index adc477e8fe..bf64b24324 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -61,7 +61,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
         if not real_urls:
             real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
             real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
-        post_date = r1(r'"?(20\d\d[-/][01]\d[-/][0123]\d)"?', html)
+        post_date = r1(r'"?(20\d\d[-/]?[01]\d[-/]?[0123]\d)"?', html)
         post_id = r1(r'/posts/([^"]+)', html)
         title = post_date + "_" + post_id
 

From c2d8729e35aa7c3b6fb709735cf7ae0f1ba6b511 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 12 Sep 2017 10:46:17 +0800
Subject: [PATCH 0502/1225] [youku]never reuse a VideoExtractor object; fix
 #2340

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index cb262bf960..5c040b680b 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -236,7 +236,7 @@ def youku_download_playlist_by_url(url, **kwargs):
         else:
             vid_list = [v['encodevid'] for v in youku_obj.video_list]
             for v in vid_list:
-                youku_obj.download_by_vid(v, **kwargs)
+                Youku().download_by_vid(v, **kwargs)
 
     elif re.match('https?://list.youku.com/show/id_', url):
         # http://list.youku.com/show/id_z2ae8ee1c837b11e18195.html

From 2652a3fe6c1dc453a04e8c9c90ee6f751ed03a29 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 12 Sep 2017 13:08:05 +0800
Subject: [PATCH 0503/1225] [extractor]add a parameter "keep_obj" to forbid
 VideoExtractor re-init after download instance method

---
 src/you_get/extractor.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index c99f46f81a..93297ebdea 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -244,5 +244,6 @@ def download(self, **kwargs):
 
             # For main_dev()
             #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
-
-        self.__init__()
+        keep_obj = kwargs.get('keep_obj', False)
+        if not keep_obj:
+            self.__init__()

From abc34c2af1922a2aced0282bd93265e94514bacc Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 12 Sep 2017 13:11:38 +0800
Subject: [PATCH 0504/1225] [youku]fix #2345

---
 src/you_get/extractors/youku.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 5c040b680b..ea8b63373f 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -62,6 +62,7 @@ def __init__(self):
 
         self.page = None
         self.video_list = None
+        self.video_next = None
         self.password = None
         self.api_data = None
         self.api_error_code = None
@@ -89,6 +90,8 @@ def youku_ups(self):
         if 'videos' in self.api_data:
             if 'list' in self.api_data['videos']:
                 self.video_list = self.api_data['videos']['list']
+            if 'next' in self.api_data['videos']:
+                self.video_next = self.api_data['videos']['next']
 
     @classmethod
     def change_cdn(cls, url):
@@ -231,12 +234,24 @@ def youku_download_playlist_by_url(url, **kwargs):
         youku_obj = Youku()
         youku_obj.url = url
         youku_obj.prepare(**kwargs)
+        total_episode = None
+        try:
+            total_episode = youku_obj.api_data['show']['episode_total']
+        except KeyError:
+            log.wtf('Cannot get total_episode for {}'.format(url))
+        next_vid = youku_obj.vid
+        for _ in range(total_episode):
+            this_extractor = Youku()
+            this_extractor.download_by_vid(next_vid, keep_obj=True, **kwargs)
+            next_vid = this_extractor.video_next['encodevid']
+        '''
         if youku_obj.video_list is None:
             log.wtf('Cannot find video list for {}'.format(url))
         else:
             vid_list = [v['encodevid'] for v in youku_obj.video_list]
             for v in vid_list:
                 Youku().download_by_vid(v, **kwargs)
+        '''
 
     elif re.match('https?://list.youku.com/show/id_', url):
         # http://list.youku.com/show/id_z2ae8ee1c837b11e18195.html

From 4c116c6f66a492fe4d6a9efa6ccef3bda5922e10 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Tue, 12 Sep 2017 13:27:42 +0800
Subject: [PATCH 0505/1225] [qq]new vid regex pattern

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index a5bb22768f..8979b4aa63 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -135,7 +135,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     else:
         content = get_content(url)
         vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
+        vid = vid[0] if vid else match1(content, r'v?id"*\s*:\s*"\s*([^"]+)"') #general fallback
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
         title = match1(content, r'"title":"([^"]+)"') if not title else title

From 657e67e1230233f8e08462fdc8568700618378f0 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Tue, 12 Sep 2017 14:01:34 +0800
Subject: [PATCH 0506/1225] fix wrong local name

thanks to @ayanamist
https://github.com/soimort/you-get/commit/bcd8d74d73f13f310a92929c5654e208ef5140eb#commitcomment-24242276
---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index cc79cfc1b4..c0e2063527 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1102,7 +1102,7 @@ def load_cookies(cookiefile):
 def set_socks_proxy(proxy):
     try:
         import socks
-        socks_proxy_addrs = socks_proxy.split(':')
+        socks_proxy_addrs = proxy.split(':')
         socks.set_default_proxy(socks.SOCKS5,
                                 socks_proxy_addrs[0],
                                 int(socks_proxy_addrs[1]))

From 854c78fd6f73219d1ef1622a6a27cb8987d04e00 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 12 Sep 2017 12:08:00 +0200
Subject: [PATCH 0507/1225] [common] remove duplicated keyword argument

- fix #2347
- fix #2348
---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c0e2063527..584b3e27cb 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1265,7 +1265,7 @@ def print_version():
         download_main(
             download, download_playlist,
             URLs, args.playlist,
-            stream_id=stream_id, output_dir=args.output_dir, merge=not args.no_merge,
+            output_dir=args.output_dir, merge=not args.no_merge,
             info_only=info_only, json_output=json_output, caption=caption,
             **extra
         )

From 388b214f0710aaecc41c6e179bbbcb25c1db7294 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 12 Sep 2017 12:37:16 +0200
Subject: [PATCH 0508/1225] [universal] quick & dirty support for MPD files
 (with BaseURL)

---
 src/you_get/extractors/universal.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 18385851e0..f70aa6ade7 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -67,6 +67,13 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         urls += re.findall(r'href="(https?://[^"]+\.png)"', page)
         urls += re.findall(r'href="(https?://[^"]+\.gif)"', page)
 
+        # MPEG-DASH MPD
+        mpd_urls = re.findall(r'src="(https?://[^"]+\.mpd)"', page)
+        for mpd_url in mpd_urls:
+            cont = get_content(mpd_url)
+            base_url = r1(r'<BaseURL>(.*)</BaseURL>', cont)
+            urls += [ r1(r'(.*/)[^/]*', mpd_url) + base_url ]
+
         # have some candy!
         candies = []
         i = 1

From 00f3dfa71f53abd495424c527b5ef3debc6fb6d2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 12 Sep 2017 12:40:10 +0200
Subject: [PATCH 0509/1225] version 0.4.909

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index bc64e7a5ad..19a154fa83 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.900'
+__version__ = '0.4.909'

From 2a250cbcb430d6991eb42fcaa4752ee39705aa7c Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 13 Sep 2017 11:09:17 +0800
Subject: [PATCH 0510/1225] [soundcloud]update client_id; move to new api
 completely; fix #2350

---
 src/you_get/extractors/soundcloud.py | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
index 6115041c8b..1a4061ffd0 100644
--- a/src/you_get/extractors/soundcloud.py
+++ b/src/you_get/extractors/soundcloud.py
@@ -6,7 +6,7 @@
 import json
 import urllib.error
 
-client_id = 'JlZIsxg2hY5WnBgtn3jfS0UYCl0K8DOg'
+client_id = 'WKcQQdEZw7Oi01KqtHWxeVSxNyRzgT8M'
 
 def soundcloud_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False):
     assert title
@@ -33,13 +33,8 @@ def soundcloud_download(url, output_dir='.', merge=True, info_only=False, **kwar
         real_url = info.get('steram_url')
     if real_url is None:
         raise Exception('Cannot get media URI for {}'.format(url))
-    real_url = '{}?client_id={}'.format(real_url, client_id)
-    try:
-        mime, ext, size = url_info(real_url)
-    except urllib.error.HTTPError as e:
-        if 401 == e.status:
-            real_url = soundcloud_i1_api(info['id'])
-            mime, ext, size = url_info(real_url)
+    real_url = soundcloud_i1_api(info['id'])
+    mime, ext, size = url_info(real_url)
     print_info(site_info, title, mime, size)
     if not info_only:
         download_urls([real_url], title, ext, size, output_dir, merge=merge)

From 51a1a1e318f4bfd463cb241d8c49346c6cad8e89 Mon Sep 17 00:00:00 2001
From: MaxwellGoblin <lispmachine@outlook.com>
Date: Wed, 13 Sep 2017 15:00:58 +0800
Subject: [PATCH 0511/1225] [qq] fix regression by 4c116c6; add guard for bad
 json

---
 src/you_get/extractors/qq.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 8979b4aa63..8ef18b0865 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -34,7 +34,10 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
         if key_json.get('key') is None:
-            log.w(key_json['msg'])
+            if part == 1:
+                log.wtf(key_json['msg'])
+            else:
+                log.w(key_json['msg'])
             break
         vkey = key_json['key']
         url = '{}{}?vkey={}'.format(host, filename, vkey)
@@ -135,7 +138,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     else:
         content = get_content(url)
         vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = vid[0] if vid else match1(content, r'v?id"*\s*:\s*"\s*([^"]+)"') #general fallback
+        vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
+        if vid is None:
+            vid = match1(content, r'id"*\s*:\s*"(.+?)"')
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title
         title = match1(content, r'"title":"([^"]+)"') if not title else title

From d1f98fa5bc1509db606b0292b9a78baea477e253 Mon Sep 17 00:00:00 2001
From: lilydjwg <lilydjwg@gmail.com>
Date: Wed, 13 Sep 2017 17:28:49 +0800
Subject: [PATCH 0512/1225] fix load_cookies local name error

---
 src/you_get/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 584b3e27cb..c9ef50b917 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1074,12 +1074,12 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
 def load_cookies(cookiefile):
     global cookies
     try:
-        cookies = cookiejar.MozillaCookieJar(a)
+        cookies = cookiejar.MozillaCookieJar(cookiefile)
         cookies.load()
     except Exception:
         import sqlite3
         cookies = cookiejar.MozillaCookieJar()
-        con = sqlite3.connect(a)
+        con = sqlite3.connect(cookiefile)
         cur = con.cursor()
         try:
             cur.execute("""SELECT host, path, isSecure, expiry, name, value

From 8831d2c44f2d115d50060ae5f8154b8d8dd27428 Mon Sep 17 00:00:00 2001
From: wwqgtxx <wwqgtxx@gmail.com>
Date: Wed, 13 Sep 2017 23:42:00 +0800
Subject: [PATCH 0513/1225] bug fix for password

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 584b3e27cb..899b2cee23 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1266,7 +1266,7 @@ def print_version():
             download, download_playlist,
             URLs, args.playlist,
             output_dir=args.output_dir, merge=not args.no_merge,
-            info_only=info_only, json_output=json_output, caption=caption,
+            info_only=info_only, json_output=json_output, caption=caption, password=args.password,
             **extra
         )
     except KeyboardInterrupt:

From cff5fb452953800ed6d47fb422c9d1a6ff4993cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 16 Sep 2017 11:46:49 +0200
Subject: [PATCH 0514/1225] version 0.4.915

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 19a154fa83..e1ff56ca47 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.909'
+__version__ = '0.4.915'

From 9e12df78076b6c127bb22c19d6003dc1bf309499 Mon Sep 17 00:00:00 2001
From: Rafael Slonik <rslonik@gmail.com>
Date: Sat, 16 Sep 2017 17:01:13 -0300
Subject: [PATCH 0515/1225] new extractor: giphy.com

---
 src/you_get/common.py           |  1 +
 src/you_get/extractors/giphy.py | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 src/you_get/extractors/giphy.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7c9fa865a3..1e7fca5b46 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -25,6 +25,7 @@
     'freesound'        : 'freesound',
     'fun'              : 'funshion',
     'google'           : 'google',
+    'giphy'            : 'giphy',
     'heavy-music'      : 'heavymusic',
     'huaban'           : 'huaban',
     'huomao'           : 'huomaotv',
diff --git a/src/you_get/extractors/giphy.py b/src/you_get/extractors/giphy.py
new file mode 100644
index 0000000000..1dd30223d2
--- /dev/null
+++ b/src/you_get/extractors/giphy.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+__all__ = ['giphy_download']
+
+from ..common import *
+import json
+
+def giphy_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    html = get_html(url)
+
+    url = list(set([
+        unicodize(str.replace(i, '\\/', '/'))
+        for i in re.findall(r'<meta property="og:video:secure_url" content="(.*?)">', html)
+    ]))
+
+    title = r1(r'<meta property="og:title" content="(.*?)">', html)
+
+    if title is None:
+      title = url[0]
+
+    type, ext, size = url_info(url[0], True)
+    size = urls_size(url)
+
+    type = "video/mp4"
+    ext = "mp4"
+
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls(url, title, ext, size, output_dir, merge=False)
+
+site_info = "Giphy.com"
+download = giphy_download
+download_playlist = playlist_not_supported('giphy')

From 12de1b482f456f9aba1e1ec48f73c204aedf9ad6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Sep 2017 14:05:56 +0200
Subject: [PATCH 0516/1225] [extractor] download best-quality DASH stream if
 FFmpeg is installed

---
 src/you_get/extractor.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 93297ebdea..198de9a496 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -194,7 +194,13 @@ def download(self, **kwargs):
                 stream_id = kwargs['stream_id']
             else:
                 # Download stream with the best quality
-                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                from .processor.ffmpeg import has_ffmpeg_installed
+                if has_ffmpeg_installed():
+                    itags = sorted(self.dash_streams,
+                                   key=lambda i: -self.dash_streams[i]['size'])
+                    stream_id = itags[0]
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
 
             if 'index' not in kwargs:
                 self.p(stream_id)

From d1c081074262234558c47476a077e08b66a5d86f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Sep 2017 17:00:10 +0200
Subject: [PATCH 0517/1225] [extractor] fix stream_id if no dash_streams exist

---
 src/you_get/extractor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 198de9a496..0df6087014 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -195,7 +195,7 @@ def download(self, **kwargs):
             else:
                 # Download stream with the best quality
                 from .processor.ffmpeg import has_ffmpeg_installed
-                if has_ffmpeg_installed():
+                if self.dash_streams and has_ffmpeg_installed():
                     itags = sorted(self.dash_streams,
                                    key=lambda i: -self.dash_streams[i]['size'])
                     stream_id = itags[0]

From 5019d9d23fac0995c33f7b2ca5818840ed3a2156 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Sep 2017 22:50:37 +0200
Subject: [PATCH 0518/1225] [bilibili] quick hack for #2377 (enforce timeout 1)

---
 src/you_get/extractors/bilibili.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 593493cb25..8ae349ef1b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -25,7 +25,7 @@ class Bilibili(VideoExtractor):
     live_api = 'http://live.bilibili.com/api/playurl?cid={}&otype=json'
     api_url = 'http://interface.bilibili.com/playurl?'
     bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
-    
+
     SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
     SEC2 = '9b288147e5474dd2aa67085f716c560d'
     stream_types = [
@@ -104,6 +104,8 @@ def download_by_vid(self, cid, bangumi, **kwargs):
                 self.parse_bili_xml(api_xml)
 
     def prepare(self, **kwargs):
+        socket.setdefaulttimeout(1) # fail fast, very speedy!
+
         self.ua = fake_headers['User-Agent']
         self.url = url_locations([self.url])[0]
         frag = urllib.parse.urlparse(self.url).fragment

From db8f59c424e1ddd5892732fef2d563cadfa81681 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 26 Sep 2017 00:37:57 +0200
Subject: [PATCH 0519/1225] [extractor] revert 12de1b4 as it's breaking
 --player

---
 src/you_get/extractor.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 0df6087014..4c9ccaa58c 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -195,12 +195,7 @@ def download(self, **kwargs):
             else:
                 # Download stream with the best quality
                 from .processor.ffmpeg import has_ffmpeg_installed
-                if self.dash_streams and has_ffmpeg_installed():
-                    itags = sorted(self.dash_streams,
-                                   key=lambda i: -self.dash_streams[i]['size'])
-                    stream_id = itags[0]
-                else:
-                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
 
             if 'index' not in kwargs:
                 self.p(stream_id)

From 9c237b2a44c01e90b5dddbd226360140e5cc4014 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Oct 2017 21:02:20 +0200
Subject: [PATCH 0520/1225] [instagram] strip query string from URL

---
 src/you_get/extractors/instagram.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 4b02ed71f1..e06eba00c2 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -5,6 +5,7 @@
 from ..common import *
 
 def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    url = r1(r'([^?]*)', url)
     html = get_html(url)
 
     vid = r1(r'instagram.com/p/([^/]+)', url)

From 6169894595f81222ca0e2d305d274d484127898e Mon Sep 17 00:00:00 2001
From: IMLink <IMLink@users.noreply.github.com>
Date: Tue, 10 Oct 2017 13:40:39 +0800
Subject: [PATCH 0521/1225] Update youku.py

fix issues #2405
---
 src/you_get/extractors/youku.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index ea8b63373f..fb8c57fa09 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -43,8 +43,12 @@ class Youku(VideoExtractor):
 
     # Last updated: 2015-11-24
     stream_types = [
+        {'id': 'mp4hd3v2', 'alias-of': 'hd3v2'},
+        {'id': 'hd3v2',    'container': 'mp4', 'video_profile': '1080P'},
         {'id': 'mp4hd3', 'alias-of': 'hd3'},
         {'id': 'hd3',    'container': 'flv', 'video_profile': '1080P'},
+        {'id': 'mp4hd2v2', 'alias-of': 'hd2v2'},
+        {'id': 'hd2v2',    'container': 'mp4', 'video_profile': '超清'},
         {'id': 'mp4hd2', 'alias-of': 'hd2'},
         {'id': 'hd2',    'container': 'flv', 'video_profile': '超清'},
         {'id': 'mp4hd',  'alias-of': 'mp4'},

From 4f97ce50f314e9110f22251dccfa0f78ebe28f9a Mon Sep 17 00:00:00 2001
From: cphilo <cphilo@qq.com>
Date: Wed, 11 Oct 2017 13:44:32 +0800
Subject: [PATCH 0522/1225] twitter download fix

---
 src/you_get/extractors/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index db01daa26c..b4f8bd9e60 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -7,9 +7,9 @@
 
 def extract_m3u(source):
     r1 = get_content(source)
-    s1 = re.findall(r'(/ext_tw_video/.*)', r1)
+    s1 = re.findall(r'(/amplify_video/.*)', r1)
     r2 = get_content('https://video.twimg.com%s' % s1[-1])
-    s2 = re.findall(r'(/ext_tw_video/.*)', r2)
+    s2 = re.findall(r'(/amplify_video/.*)', r2)
     return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):

From 56065c0a4881f89c93f840cf0afe92efd964701b Mon Sep 17 00:00:00 2001
From: jacoz <zjcong1@gmail.com>
Date: Fri, 13 Oct 2017 16:30:37 +0800
Subject: [PATCH 0523/1225] [pptv] fix #2422

---
 src/you_get/extractors/pptv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index 9e55ac6c09..17503c1cd4 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -174,7 +174,7 @@ def make_url(stream):
     src = []
     for i, seg in enumerate(stream['segs']):
         url = 'http://{}/{}/{}?key={}&k={}'.format(host, i, rid, key, key_expr)
-        url += '&fpp.ver=1.3.0.4&type=web.fpp'
+        url += '&fpp.ver=1.3.0.4&type='
         src.append(url)
     return src
 
@@ -198,7 +198,7 @@ def prepare(self, **kwargs):
         if not self.vid:
             raise('Cannot find id')
         api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid)
-        api_url += '?type=web.fpp&version=4'
+        api_url += '?appplt=flp&appid=pptv.flashplayer.vod&appver=3.4.2.28&type=&version=4'
         dom = parseString(get_content(api_url))
         self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom)
         xml_streams = merge_meta(m_items, m_streams, m_segs)

From 003c7256ee4d465953847bb4f1e164a919c51165 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 15:20:06 +0200
Subject: [PATCH 0524/1225] [twitter] fix regression

---
 src/you_get/extractors/twitter.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index b4f8bd9e60..9a3822d1f2 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -7,9 +7,11 @@
 
 def extract_m3u(source):
     r1 = get_content(source)
-    s1 = re.findall(r'(/amplify_video/.*)', r1)
+    s1 = re.findall(r'(/ext_tw_video/.*)', r1)
+    s1 += re.findall(r'(/amplify_video/.*)', r1)
     r2 = get_content('https://video.twimg.com%s' % s1[-1])
-    s2 = re.findall(r'(/amplify_video/.*)', r2)
+    s2 = re.findall(r'(/ext_tw_video/.*)', r2)
+    s2 += re.findall(r'(/amplify_video/.*)', r2)
     return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):

From 74291bf5f7cfd9830d20c7cc10d02a87ed0c9b65 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 15:58:29 +0200
Subject: [PATCH 0525/1225] [qq] fix #2409

---
 src/you_get/extractors/qq.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 8ef18b0865..b675839e75 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -13,8 +13,6 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     info = get_content(info_api)
     video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
 
-    if video_json['exem'] != 0:
-        log.wtf(video_json['msg'])
     fn_pre = video_json['vl']['vi'][0]['lnk']
     title = video_json['vl']['vi'][0]['ti']
     host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
@@ -63,7 +61,7 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
     real_url = real_url.replace('\/', '/')
 
     ksong_mid = json_data['data']['ksong_mid']
-    lyric_url = 'http://cgi.kg.qq.com/fcgi-bin/fcg_lyric?jsonpCallback=jsopgetlrcdata&outCharset=utf-8&ksongmid=' + ksong_mid 
+    lyric_url = 'http://cgi.kg.qq.com/fcgi-bin/fcg_lyric?jsonpCallback=jsopgetlrcdata&outCharset=utf-8&ksongmid=' + ksong_mid
     lyric_data = get_content(lyric_url)
     lyric_string = lyric_data[len('jsopgetlrcdata('):-1]
     lyric_json = json.loads(lyric_string)

From 4b56e46d2a7be70d87ff5e5ac125133fee4fed9a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 16:31:51 +0200
Subject: [PATCH 0526/1225] [bilibili] support "watchlater" URLs (per #2424)

---
 src/you_get/extractors/bilibili.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 8ae349ef1b..4642a6215c 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -106,6 +106,11 @@ def download_by_vid(self, cid, bangumi, **kwargs):
     def prepare(self, **kwargs):
         socket.setdefaulttimeout(1) # fail fast, very speedy!
 
+        # handle "watchlater" URLs
+        if '/watchlater/' in self.url:
+            aid = re.search(r'av(\d+)', self.url).group(1)
+            self.url = 'http://www.bilibili.com/video/av{}/'.format(aid)
+
         self.ua = fake_headers['User-Agent']
         self.url = url_locations([self.url])[0]
         frag = urllib.parse.urlparse(self.url).fragment
@@ -125,6 +130,7 @@ def prepare(self, **kwargs):
                 self.title = '{} {}'.format(self.title, subtitle)
         except Exception:
             pass
+
         if 'bangumi.bilibili.com/movie' in self.url:
             self.movie_entry(**kwargs)
         elif 'bangumi.bilibili.com' in self.url:

From 77542320f093311fa54b5450a44ee07ef207f3dc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 17:18:53 +0200
Subject: [PATCH 0527/1225] [tudou] fix #2427

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index fb8c57fa09..e72535f8de 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -148,7 +148,7 @@ def prepare(self, **kwargs):
                     log.wtf('Cannot fetch vid')
 
         if kwargs.get('src') and kwargs['src'] == 'tudou':
-            self.ccode = '0402'
+            self.ccode = '050F'
 
         if kwargs.get('password') and kwargs['password']:
             self.password_protected = True

From 31200781b22d1c1b740a73c587cb1dcc2a962ef4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 17:56:08 +0200
Subject: [PATCH 0528/1225] [qq] fix #2353

---
 src/you_get/extractors/qq.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index b675839e75..c108d422f9 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -27,7 +27,10 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     part_urls= []
     total_size = 0
     for part in range(1, seg_cnt+1):
-        filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+        if seg_cnt == 1 and video_json['vl']['vi'][0]['vh'] <= 480:
+            filename = fn_pre + '.mp4'
+        else:
+            filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
         key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])

From 688f7fb6d5e722897c8dc84904dd2a4e84e7bbe7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 21:36:50 +0200
Subject: [PATCH 0529/1225] [xiami] add support for MVs, fix #2426

---
 src/you_get/extractors/xiami.py | 35 ++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
index 2d362b2703..16656adb17 100644
--- a/src/you_get/extractors/xiami.py
+++ b/src/you_get/extractors/xiami.py
@@ -149,7 +149,37 @@ def xiami_download_album(aid, output_dir='.', info_only=False):
 
         track_nr += 1
 
-def xiami_download(url, output_dir='.', info_only=False, **kwargs):
+def xiami_download_mv(url, output_dir='.', merge=True, info_only=False):
+    # FIXME: broken merge
+    page = get_content(url, headers=fake_headers)
+    title = re.findall('<title>([^<]+)', page)[0]
+    vid, uid = re.findall(r'vid:"(\d+)",uid:"(\d+)"', page)[0]
+    api_url = 'http://cloud.video.taobao.com/videoapi/info.php?vid=%s&uid=%s' % (vid, uid)
+    result = get_content(api_url, headers=fake_headers)
+    doc = parseString(result)
+    video_url = doc.getElementsByTagName("video_url")[-1].firstChild.nodeValue
+    length = int(doc.getElementsByTagName("length")[-1].firstChild.nodeValue)
+
+    v_urls = []
+    k_start = 0
+    total_size = 0
+    while True:
+        k_end = k_start + 20000000
+        if k_end >= length: k_end = length - 1
+        v_url = video_url + '/start_%s/end_%s/1.flv' % (k_start, k_end)
+        try:
+            _, ext, size = url_info(v_url)
+        except:
+            break
+        v_urls.append(v_url)
+        total_size += size
+        k_start = k_end + 1
+
+    print_info(site_info, title, ext, total_size)
+    if not info_only:
+        download_urls(v_urls, title, ext, total_size, output_dir, merge=merge, headers=fake_headers)
+
+def xiami_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 #albums
     if re.match(r'http://www.xiami.com/album/\d+', url):
         id = r1(r'http://www.xiami.com/album/(\d+)', url)
@@ -177,6 +207,9 @@ def xiami_download(url, output_dir='.', info_only=False, **kwargs):
         id = r1(r'http://www.xiami.com/song/detail/id/(\d+)', url)
         xiami_download_song(id, output_dir, info_only)
 
+    if re.match('http://www.xiami.com/mv', url):
+        xiami_download_mv(url, output_dir, merge=merge, info_only=info_only)
+
 site_info = "Xiami.com"
 download = xiami_download
 download_playlist = playlist_not_supported("xiami")

From 871d725584a2a016c18b2e02ddf214e23dd53555 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 22:38:16 +0200
Subject: [PATCH 0530/1225] [youku] fix #2412 and close #2429

---
 src/you_get/extractors/youku.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index e72535f8de..1ae49e7537 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -52,6 +52,7 @@ class Youku(VideoExtractor):
         {'id': 'mp4hd2', 'alias-of': 'hd2'},
         {'id': 'hd2',    'container': 'flv', 'video_profile': '超清'},
         {'id': 'mp4hd',  'alias-of': 'mp4'},
+        {'id': 'mp4sd',  'alias-of': 'mp4'},
         {'id': 'mp4',    'container': 'mp4', 'video_profile': '高清'},
         {'id': 'flvhd',  'container': 'flv', 'video_profile': '标清'},
         {'id': 'flv',    'container': 'flv', 'video_profile': '标清'},

From d24545e66b83f8f79b59c9d19e1213144253295f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 23:09:50 +0200
Subject: [PATCH 0531/1225] [youku] update stream_types (2017-10-13)

---
 src/you_get/extractors/youku.py | 34 ++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 1ae49e7537..d3e4c22d66 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -41,22 +41,26 @@ class Youku(VideoExtractor):
     mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
     dispatcher_url = 'vali.cp31.ott.cibntv.net'
 
-    # Last updated: 2015-11-24
+    # Last updated: 2017-10-13
     stream_types = [
-        {'id': 'mp4hd3v2', 'alias-of': 'hd3v2'},
-        {'id': 'hd3v2',    'container': 'mp4', 'video_profile': '1080P'},
-        {'id': 'mp4hd3', 'alias-of': 'hd3'},
-        {'id': 'hd3',    'container': 'flv', 'video_profile': '1080P'},
-        {'id': 'mp4hd2v2', 'alias-of': 'hd2v2'},
-        {'id': 'hd2v2',    'container': 'mp4', 'video_profile': '超清'},
-        {'id': 'mp4hd2', 'alias-of': 'hd2'},
-        {'id': 'hd2',    'container': 'flv', 'video_profile': '超清'},
-        {'id': 'mp4hd',  'alias-of': 'mp4'},
-        {'id': 'mp4sd',  'alias-of': 'mp4'},
-        {'id': 'mp4',    'container': 'mp4', 'video_profile': '高清'},
-        {'id': 'flvhd',  'container': 'flv', 'video_profile': '标清'},
-        {'id': 'flv',    'container': 'flv', 'video_profile': '标清'},
-        {'id': '3gphd',  'container': 'mp4', 'video_profile': '标清（3GP）'},
+        {'id': 'hd3',      'container': 'flv', 'video_profile': '1080P'},
+        {'id': 'hd3v2',    'container': 'flv', 'video_profile': '1080P'},
+        {'id': 'mp4hd3',   'container': 'mp4', 'video_profile': '1080P'},
+        {'id': 'mp4hd3v2', 'container': 'mp4', 'video_profile': '1080P'},
+
+        {'id': 'hd2',      'container': 'flv', 'video_profile': '超清'},
+        {'id': 'hd2v2',    'container': 'flv', 'video_profile': '超清'},
+        {'id': 'mp4hd2',   'container': 'mp4', 'video_profile': '超清'},
+        {'id': 'mp4hd2v2', 'container': 'mp4', 'video_profile': '超清'},
+
+        {'id': 'flvhd',    'container': 'flv', 'video_profile': '高清'},
+        {'id': 'mp4hd',    'container': 'mp4', 'video_profile': '高清'},
+        {'id': '3gphd',    'container': 'mp4', 'video_profile': '高清'},
+
+        {'id': 'mp4sd',    'container': 'mp4', 'video_profile': '标清'},
+        # obsolete?
+        {'id': 'flv',      'container': 'flv', 'video_profile': '标清'},
+        {'id': 'mp4',      'container': 'mp4', 'video_profile': '标清'},
     ]
 
     def __init__(self):

From cf0abaeb941d179a7cb4e6a120a992c9e4a771a2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 23:37:55 +0200
Subject: [PATCH 0532/1225] update README

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 57f49a6875..5072fd7707 100644
--- a/README.md
+++ b/README.md
@@ -394,7 +394,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
 | Sohu<br/>搜狐视频 | <http://tv.sohu.com/> |✓| | |
 | **Tudou<br/>土豆** | <http://www.tudou.com/> |✓| | |
-| 虾米     | <http://www.xiami.com/>        | | |✓|
+| 虾米     | <http://www.xiami.com/>        |✓| |✓|
 | 阳光卫视 | <http://www.isuntv.com/>       |✓| | |
 | **音悦Tai** | <http://www.yinyuetai.com/> |✓| | |
 | **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |

From 581b15a38628a40bb6b58884a87f840c6069f08a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 23:43:07 +0200
Subject: [PATCH 0533/1225] I love beer!

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5072fd7707..403a288f6d 100644
--- a/README.md
+++ b/README.md
@@ -448,6 +448,6 @@ We only ship the code here, and how you are going to use it is left to your own
 
 ## Authors
 
-Made by [@soimort](https://github.com/soimort), who is in turn powered by :coffee:, :pizza: and :ramen:.
+Made by [@soimort](https://github.com/soimort), who is in turn powered by :coffee:, :beer: and :ramen:.
 
 You can find the [list of all contributors](https://github.com/soimort/you-get/graphs/contributors) here.

From 2c238ae683f295d8036837c9d4de2cc39871115b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 23:45:33 +0200
Subject: [PATCH 0534/1225] version 0.4.939

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e1ff56ca47..63d93414ab 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.915'
+__version__ = '0.4.939'

From c2de0bdc38cf5044548176b975ce32c87016fb6b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 13 Oct 2017 23:59:07 +0200
Subject: [PATCH 0535/1225] [youku] update stream_types

---
 src/you_get/extractors/youku.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d3e4c22d66..65a60f580f 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -53,9 +53,10 @@ class Youku(VideoExtractor):
         {'id': 'mp4hd2',   'container': 'mp4', 'video_profile': '超清'},
         {'id': 'mp4hd2v2', 'container': 'mp4', 'video_profile': '超清'},
 
-        {'id': 'flvhd',    'container': 'flv', 'video_profile': '高清'},
         {'id': 'mp4hd',    'container': 'mp4', 'video_profile': '高清'},
-        {'id': '3gphd',    'container': 'mp4', 'video_profile': '高清'},
+        # not really equivalent to mp4hd
+        {'id': 'flvhd',    'container': 'flv', 'video_profile': '渣清'},
+        {'id': '3gphd',    'container': 'mp4', 'video_profile': '渣清'},
 
         {'id': 'mp4sd',    'container': 'mp4', 'video_profile': '标清'},
         # obsolete?

From 46fc02fa06dd2bdca637a8fda66fccc5491e4410 Mon Sep 17 00:00:00 2001
From: ayanamist <ayanamist@gmail.com>
Date: Mon, 16 Oct 2017 14:02:55 +0800
Subject: [PATCH 0536/1225] modify bilibili title fetch

---
 src/you_get/extractors/bilibili.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 4642a6215c..10154c3315 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -123,13 +123,17 @@ def prepare(self, **kwargs):
                 self.url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, page)
         self.referer = self.url
         self.page = get_content(self.url)
-        try:
-            self.title = re.search(r'<h1\s*title="([^"]+)"', self.page).group(1)
-            if 'subtitle' in kwargs:
-                subtitle = kwargs['subtitle']
-                self.title = '{} {}'.format(self.title, subtitle)
-        except Exception:
-            pass
+
+        m = re.search(r'<h1\s*title="([^"]+)"', self.page)
+        if m is not None:
+            self.title = m.group(1)
+        if self.title is None:
+            m = re.search(r'<meta property="og:title" content="([^"]+)">', self.page)
+            if m is not None:
+                self.title = m.group(1)
+        if 'subtitle' in kwargs:
+            subtitle = kwargs['subtitle']
+            self.title = '{} {}'.format(self.title, subtitle)
 
         if 'bangumi.bilibili.com/movie' in self.url:
             self.movie_entry(**kwargs)

From e88364bf56d3334671126b4c094db24f0fc8763a Mon Sep 17 00:00:00 2001
From: ayanamist <ayanamist@gmail.com>
Date: Thu, 5 Oct 2017 11:54:06 +0800
Subject: [PATCH 0537/1225] fix urlopen_with_retry may return None

---
 src/you_get/common.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1e7fca5b46..f56f85e373 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -342,14 +342,19 @@ def get_location(url):
     return response.geturl()
 
 def urlopen_with_retry(*args, **kwargs):
-    for i in range(2):
+    retry_time = 3
+    for i in range(retry_time):
         try:
             return request.urlopen(*args, **kwargs)
-        except socket.timeout:
+        except socket.timeout as e:
             logging.debug('request attempt %s timeout' % str(i + 1))
+            if i + 1 == retry_time:
+                raise e
 # try to tackle youku CDN fails
         except error.HTTPError as http_error:
             logging.debug('HTTP Error with code{}'.format(http_error.code))
+            if i + 1 == retry_time:
+                raise http_error
 
 def get_content(url, headers={}, decoded=True):
     """Gets the content of a URL via sending a HTTP GET request.

From ed3c952f3419524e57661c0c43bcbe4a7c01027d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 18 Oct 2017 11:53:16 +0200
Subject: [PATCH 0538/1225] [bilibili] fail fast, but not too fast (close
 #2437)

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 4642a6215c..73f8281779 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -104,7 +104,7 @@ def download_by_vid(self, cid, bangumi, **kwargs):
                 self.parse_bili_xml(api_xml)
 
     def prepare(self, **kwargs):
-        socket.setdefaulttimeout(1) # fail fast, very speedy!
+        socket.setdefaulttimeout(2) # fail fast, very speedy!
 
         # handle "watchlater" URLs
         if '/watchlater/' in self.url:

From 10bf672473d1569e10a18a8c574d59e40ab81378 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 18 Oct 2017 12:03:56 +0200
Subject: [PATCH 0539/1225] [icourses] fix typo, close #2433

---
 src/you_get/extractors/icourses.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index 0140281a26..ec70f64cba 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -18,7 +18,7 @@ def icourses_download(url, output_dir='.', **kwargs):
         hit = re.search(r'id=(\d+)&courseId=(\d+)', url)
         url = 'http://www.icourses.cn/jpk/changeforVideo.action?resId={}&courseId={}'.format(hit.group(1), hit.group(2))
     if re.match(r'http://www.icourses.cn/coursestatic/course_(\d+).html', url):
-        raise Exception('You can donwload it with -l flag')
+        raise Exception('You can download it with -l flag')
     icourses_parser = ICousesExactor(url=url)
     icourses_parser.basic_extract()
     title = icourses_parser.title

From c3b31c5d3147473061a7a51f7d13fbcbe8831e2f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 18 Oct 2017 12:15:31 +0200
Subject: [PATCH 0540/1225] [bilibili] respect --timeout

---
 src/you_get/extractors/bilibili.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1dc7438c16..f75397d6a7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -104,7 +104,8 @@ def download_by_vid(self, cid, bangumi, **kwargs):
                 self.parse_bili_xml(api_xml)
 
     def prepare(self, **kwargs):
-        socket.setdefaulttimeout(2) # fail fast, very speedy!
+        if socket.getdefaulttimeout() == 600: # no timeout specified
+            socket.setdefaulttimeout(2) # fail fast, very speedy!
 
         # handle "watchlater" URLs
         if '/watchlater/' in self.url:

From 6c9c15e69f415c5bed9bc3c697bb18012a73f6a3 Mon Sep 17 00:00:00 2001
From: ayanamist <ayanamist@gmail.com>
Date: Thu, 5 Oct 2017 11:54:36 +0800
Subject: [PATCH 0541/1225] fix bar display under windows terminal

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f56f85e373..fe8fbbd028 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -643,7 +643,7 @@ def __init__(self, total_size, total_pieces = 1):
         # 38 is the size of all statically known size in self.bar
         total_str = '%5s' % round(self.total_size / 1048576, 1)
         total_str_width = max(len(total_str), 5)
-        self.bar_size = self.term_size - 27 - 2*total_pieces_len - 2*total_str_width
+        self.bar_size = self.term_size - 28 - 2*total_pieces_len - 2*total_str_width
         self.bar = '{:>4}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % (
             total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len)
 

From ac6b97618f24ae744a1fcce9eecbcc855e1c5901 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Oct 2017 21:02:53 +0200
Subject: [PATCH 0542/1225] [universal] case-insensitive baby

---
 src/you_get/extractors/universal.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index f70aa6ade7..633cf55b4b 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -63,9 +63,9 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             urls += [url.replace('\\\\/', '/') for url in q_urls]
 
         # a link href to an image is often an interesting one
-        urls += re.findall(r'href="(https?://[^"]+\.jpe?g)"', page)
-        urls += re.findall(r'href="(https?://[^"]+\.png)"', page)
-        urls += re.findall(r'href="(https?://[^"]+\.gif)"', page)
+        urls += re.findall(r'href="(https?://[^"]+\.jpe?g)"', page, re.I)
+        urls += re.findall(r'href="(https?://[^"]+\.png)"', page, re.I)
+        urls += re.findall(r'href="(https?://[^"]+\.gif)"', page, re.I)
 
         # MPEG-DASH MPD
         mpd_urls = re.findall(r'src="(https?://[^"]+\.mpd)"', page)

From 36192cea44b81834430aa4a422ba7b2f999993a2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 Oct 2017 21:07:37 +0100
Subject: [PATCH 0543/1225] [acfun] fix active single-p title

---
 src/you_get/extractors/acfun.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 0ed8cf9269..a5ea765920 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -117,7 +117,9 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
     vid = r1('data-vid="(\d+)"', html)
     up = r1('data-name="([^"]+)"', html)
-    title = title + ' - ' + up
+    p_title = r1('active">([^<]+)', html)
+    title = '%s (%s)' % (title, up)
+    if p_title: title = '%s - %s' % (title, p_title)
     acfun_download_by_vid(vid, title,
                           output_dir=output_dir,
                           merge=merge,

From fbd492220e3455db28c9389e4844d58a7967be8e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 31 Oct 2017 19:02:44 +0100
Subject: [PATCH 0544/1225] [tumblr] ignore bad URLs

---
 src/you_get/extractors/tumblr.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 0c59f25a01..5817e5483c 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -32,15 +32,17 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
             quality = int(r1(r'^tumblr_.+_(\d+)$', title))
             ext = filename.split('.')[-1]
-            size = int(get_head(url)['Content-Length'])
-            if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
-                tuggles[tumblr_id] = {
-                    'title': title,
-                    'url': url,
-                    'quality': quality,
-                    'ext': ext,
-                    'size': size,
-                }
+            try:
+                size = int(get_head(url)['Content-Length'])
+                if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
+                    tuggles[tumblr_id] = {
+                        'title': title,
+                        'url': url,
+                        'quality': quality,
+                        'ext': ext,
+                        'size': size,
+                    }
+            except: pass
 
         if tuggles:
             size = sum([tuggles[t]['size'] for t in tuggles])

From 289c42115f8adcf8b5fb08a4c2f3c7d876acf333 Mon Sep 17 00:00:00 2001
From: jhc888007 <jhc888007@163@com>
Date: Mon, 6 Nov 2017 15:42:40 +0800
Subject: [PATCH 0545/1225] qq video does not work well any more

---
 src/you_get/extractors/qq.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index c108d422f9..91ef7301ee 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -27,10 +27,11 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     part_urls= []
     total_size = 0
     for part in range(1, seg_cnt+1):
-        if seg_cnt == 1 and video_json['vl']['vi'][0]['vh'] <= 480:
-            filename = fn_pre + '.mp4'
-        else:
-            filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+        #if seg_cnt == 1 and video_json['vl']['vi'][0]['vh'] <= 480:
+        #    filename = fn_pre + '.mp4'
+        #else:
+        #    filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+        filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
         key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])

From 04b5f9f95adf4f584b26417bff19950cc7a46ef4 Mon Sep 17 00:00:00 2001
From: hcwhan <hcwhan@gmail.com>
Date: Wed, 8 Nov 2017 14:38:44 +0800
Subject: [PATCH 0546/1225] add header and update douyutv api fix StatusCode
 403

---
 src/you_get/extractors/douyutv.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index ae719e0d40..3b64066afc 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -40,31 +40,34 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
         douyutv_video_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return
 
-    html = get_content(url)
+    headers = {
+        'user-agent': 'Mozilla/5.0 (iPad; CPU OS 8_1_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B466 Safari/600.1.4'
+    }
+    html = get_content(url, headers)
     room_id_patt = r'"room_id"\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
     if room_id == "0":
         room_id = url[url.rfind('/')+1:]
 
-    json_request_url = "http://m.douyu.com/html5/live?roomId=%s" % room_id
-    content = get_content(json_request_url)
+    api_url = "http://www.douyutv.com/api/v1/"
+    args = "room/%s?aid=wp&client_sys=wp&time=%d" % (room_id, int(time.time()))
+    auth_md5 = (args + "zNzMV1y4EMxOHS6I5WKm").encode("utf-8")
+    auth_str = hashlib.md5(auth_md5).hexdigest()
+    json_request_url = "%s%s&auth=%s" % (api_url, args, auth_str)
+
+    content = get_content(json_request_url, headers)
     json_content = json.loads(content)
     data = json_content['data']
     server_status = json_content.get('error',0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
 
-    room_info_url = "http://open.douyucdn.cn/api/RoomApi/room/%s" % room_id
-    room_info_content = get_content(room_info_url)
-    room_info_obj = json.loads(room_info_content)
-    room_info_data = room_info_obj.get('data')
-
-    title = room_info_data.get('room_name')
-    show_status = room_info_data.get('room_status')
+    title = data.get('room_name')
+    show_status = data.get('show_status')
     if show_status is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
-    real_url = data.get('hls_url')
+    real_url = data.get('rtmp_url') + '/' + data.get('rtmp_live')
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From ab8683baf93d4a679ff8b7d9c69b400f4318e11c Mon Sep 17 00:00:00 2001
From: shitfly <shitfly@shitflydeMacBook-Pro.local>
Date: Wed, 8 Nov 2017 17:10:52 +0800
Subject: [PATCH 0547/1225] [sohu]add https

---
 src/you_get/extractors/sohu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 9e06892176..58eb1ac76f 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -27,7 +27,7 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
     assert vid
 
-    if re.match(r'http://tv.sohu.com/', url):
+    if re.match(r'http[s]://tv.sohu.com/', url):
         if extractor_proxy:
             set_proxy(tuple(extractor_proxy.split(":")))
         info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))

From 3f3d07b50564a89ea0eb5ecfd27ce2fbe0b37dd8 Mon Sep 17 00:00:00 2001
From: shitfly <shitfly@shitflydeMacBook-Pro.local>
Date: Thu, 9 Nov 2017 11:27:38 +0800
Subject: [PATCH 0548/1225] [qq]fix vid

---
 src/you_get/extractors/qq.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index c108d422f9..8ea74fdd67 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -138,8 +138,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = vid
     else:
         content = get_content(url)
-        vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = vid[0] if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
+        #vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
+        vid = url.split('/')[-1].split('.')[0] #https://v.qq.com/x/cover/ps6mnfqyrfo7es3/q0181hpdvo5.html?
+        vid = vid if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
         if vid is None:
             vid = match1(content, r'id"*\s*:\s*"(.+?)"')
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)

From ce67a7d122eb40fc64da5c99e63d0e506ef093c7 Mon Sep 17 00:00:00 2001
From: Ein Verne <einverne@gmail.com>
Date: Fri, 10 Nov 2017 13:59:03 +0800
Subject: [PATCH 0549/1225] Fix miaopai download failed

Signed-off-by: Ein Verne <einverne@gmail.com>
---
 src/you_get/extractors/yixia.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index 13b56a3f15..a37c359c7a 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -51,14 +51,14 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         yixia_download_by_scid = yixia_miaopai_download_by_scid
         site_info = "Yixia Miaopai"
         
-        if re.match(r'http://www.miaopai.com/show/channel/.+', url):  #PC
-            scid = match1(url, r'http://www.miaopai.com/show/channel/(.+)\.htm')
-        elif re.match(r'http://www.miaopai.com/show/.+', url):  #PC
-            scid = match1(url, r'http://www.miaopai.com/show/(.+)\.htm')
-        elif re.match(r'http://m.miaopai.com/show/channel/.+', url):  #Mobile
-            scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)\.htm')
+        if re.match(r'https?://www.miaopai.com/show/channel/.+', url):  #PC
+            scid = match1(url, r'https?://www.miaopai.com/show/channel/(.+)\.htm')
+        elif re.match(r'https?://www.miaopai.com/show/.+', url):  #PC
+            scid = match1(url, r'https?://www.miaopai.com/show/(.+)\.htm')
+        elif re.match(r'https?://m.miaopai.com/show/channel/.+', url):  #Mobile
+            scid = match1(url, r'https?://m.miaopai.com/show/channel/(.+)\.htm')
             if scid == None :
-                scid = match1(url, r'http://m.miaopai.com/show/channel/(.+)')
+                scid = match1(url, r'https?://m.miaopai.com/show/channel/(.+)')
 
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu
         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid

From 4f4a7d2d73717ef50372a5e78948861e47794d3d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 11 Nov 2017 05:24:02 +0100
Subject: [PATCH 0550/1225] [twitter] for tomorrow, we hunt

---
 src/you_get/extractors/twitter.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 9a3822d1f2..04fd8773d1 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -62,7 +62,8 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         i = json.loads(unescape_html(data_config))
         if 'video_url' in i:
             source = i['video_url']
-            if not item_id: page_title = i['tweet_id']
+            item_id = i['tweet_id']
+            page_title = "{} [{}]".format(screen_name, item_id)
         elif 'playlist' in i:
             source = i['playlist'][0]['source']
             if not item_id: page_title = i['playlist'][0]['contentId']
@@ -70,7 +71,8 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             vmap_url = i['vmap_url']
             vmap = get_content(vmap_url)
             source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
-            if not item_id: page_title = i['tweet_id']
+            item_id = i['tweet_id']
+            page_title = "{} [{}]".format(screen_name, item_id)
         elif 'scribe_playlist_url' in i:
             scribe_playlist_url = i['scribe_playlist_url']
             return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)

From b327a5ef9622500b79659464e2614b94002e0db2 Mon Sep 17 00:00:00 2001
From: iawia002 <z2d@jifangcheng.com>
Date: Sun, 12 Nov 2017 16:06:22 +0800
Subject: [PATCH 0551/1225] pep8 update for common.py

---
 src/you_get/common.py | 588 ++++++++++++++++++++++++++++--------------
 1 file changed, 397 insertions(+), 191 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f56f85e373..1b9901d501 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1,5 +1,25 @@
 #!/usr/bin/env python
 
+import os
+import re
+import sys
+import time
+import json
+import socket
+import locale
+import logging
+import argparse
+from http import cookiejar
+from importlib import import_module
+from urllib import request, parse, error
+
+from .version import __version__
+from .util import log, term
+from .util.git import get_version
+from .util.strings import get_filename, unescape_html
+from . import json_output as json_output_
+
+
 SITES = {
     '163'              : 'netease',
     '56'               : 'w56',
@@ -101,25 +121,6 @@
     'zhanqi'           : 'zhanqi',
 }
 
-import json
-import locale
-import logging
-import os
-import re
-import socket
-import sys
-import time
-from urllib import request, parse, error
-from http import cookiejar
-from importlib import import_module
-import argparse
-
-from .version import __version__
-from .util import log, term
-from .util.git import get_version
-from .util.strings import get_filename, unescape_html
-from . import json_output as json_output_
-
 dry_run = False
 json_output = False
 force = False
@@ -129,11 +130,11 @@
 output_filename = None
 
 fake_headers = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0'
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0',  # noqa
 }
 
 if sys.stdout.isatty():
@@ -141,9 +142,10 @@
 else:
     default_encoding = locale.getpreferredencoding().lower()
 
+
 def rc4(key, data):
-#all encryption algo should work on bytes
-    assert type(key)==type(data) and type(key) == type(b'')
+    # all encryption algo should work on bytes
+    assert type(key) == type(data) and type(key) == type(b'')
     state = list(range(256))
     j = 0
     for i in range(256):
@@ -165,6 +167,7 @@ def rc4(key, data):
 
     return bytes(out_list)
 
+
 def general_m3u8_extractor(url, headers={}):
     m3u8_list = get_content(url, headers=headers).split('\n')
     urls = []
@@ -178,16 +181,21 @@ def general_m3u8_extractor(url, headers={}):
                 urls.append(seg_url)
     return urls
 
+
 def maybe_print(*s):
-    try: print(*s)
-    except: pass
+    try:
+        print(*s)
+    except:
+        pass
+
 
 def tr(s):
     if default_encoding == 'utf-8':
         return s
     else:
         return s
-        #return str(s.encode('utf-8'))[2:-1]
+        # return str(s.encode('utf-8'))[2:-1]
+
 
 # DEPRECATED in favor of match1()
 def r1(pattern, text):
@@ -195,6 +203,7 @@ def r1(pattern, text):
     if m:
         return m.group(1)
 
+
 # DEPRECATED in favor of match1()
 def r1_of(patterns, text):
     for p in patterns:
@@ -202,6 +211,7 @@ def r1_of(patterns, text):
         if x:
             return x
 
+
 def match1(text, *patterns):
     """Scans through a string for substrings matched some patterns (first-subgroups only).
 
@@ -229,6 +239,7 @@ def match1(text, *patterns):
                 ret.append(match.group(1))
         return ret
 
+
 def matchall(text, patterns):
     """Scans through a string for substrings matched some patterns.
 
@@ -247,11 +258,13 @@ def matchall(text, patterns):
 
     return ret
 
+
 def launch_player(player, urls):
     import subprocess
     import shlex
     subprocess.call(shlex.split(player) + list(urls))
 
+
 def parse_query_param(url, param):
     """Parses the query string of a URL and returns the value of a parameter.
 
@@ -268,8 +281,14 @@ def parse_query_param(url, param):
     except:
         return None
 
+
 def unicodize(text):
-    return re.sub(r'\\u([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])', lambda x: chr(int(x.group(0)[2:], 16)), text)
+    return re.sub(
+        r'\\u([0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f][0-9A-Fa-f])',
+        lambda x: chr(int(x.group(0)[2:], 16)),
+        text
+    )
+
 
 # DEPRECATED in favor of util.legitimize()
 def escape_file_path(path):
@@ -279,6 +298,7 @@ def escape_file_path(path):
     path = path.replace('?', '-')
     return path
 
+
 def ungzip(data):
     """Decompresses data for Content-Encoding: gzip.
     """
@@ -288,6 +308,7 @@ def ungzip(data):
     f = gzip.GzipFile(fileobj=buffer)
     return f.read()
 
+
 def undeflate(data):
     """Decompresses data for Content-Encoding: deflate.
     (the zlib compression is used.)
@@ -296,8 +317,9 @@ def undeflate(data):
     decompressobj = zlib.decompressobj(-zlib.MAX_WBITS)
     return decompressobj.decompress(data)+decompressobj.flush()
 
+
 # DEPRECATED in favor of get_content()
-def get_response(url, faker = False):
+def get_response(url, faker=False):
     logging.debug('get_response: %s' % url)
 
     # install cookies
@@ -306,7 +328,9 @@ def get_response(url, faker = False):
         request.install_opener(opener)
 
     if faker:
-        response = request.urlopen(request.Request(url, headers = fake_headers), None)
+        response = request.urlopen(
+            request.Request(url, headers=fake_headers), None
+        )
     else:
         response = request.urlopen(url)
 
@@ -318,13 +342,15 @@ def get_response(url, faker = False):
     response.data = data
     return response
 
+
 # DEPRECATED in favor of get_content()
-def get_html(url, encoding = None, faker = False):
+def get_html(url, encoding=None, faker=False):
     content = get_response(url, faker).data
     return str(content, 'utf-8', 'ignore')
 
+
 # DEPRECATED in favor of get_content()
-def get_decoded_html(url, faker = False):
+def get_decoded_html(url, faker=False):
     response = get_response(url, faker)
     data = response.data
     charset = r1(r'charset=([\w-]+)', response.headers['content-type'])
@@ -333,6 +359,7 @@ def get_decoded_html(url, faker = False):
     else:
         return data
 
+
 def get_location(url):
     logging.debug('get_location: %s' % url)
 
@@ -341,6 +368,7 @@ def get_location(url):
     # not to do that
     return response.geturl()
 
+
 def urlopen_with_retry(*args, **kwargs):
     retry_time = 3
     for i in range(retry_time):
@@ -350,12 +378,13 @@ def urlopen_with_retry(*args, **kwargs):
             logging.debug('request attempt %s timeout' % str(i + 1))
             if i + 1 == retry_time:
                 raise e
-# try to tackle youku CDN fails
+        # try to tackle youku CDN fails
         except error.HTTPError as http_error:
             logging.debug('HTTP Error with code{}'.format(http_error.code))
             if i + 1 == retry_time:
                 raise http_error
 
+
 def get_content(url, headers={}, decoded=True):
     """Gets the content of a URL via sending a HTTP GET request.
 
@@ -387,7 +416,9 @@ def get_content(url, headers={}, decoded=True):
 
     # Decode the response body
     if decoded:
-        charset = match1(response.getheader('Content-Type'), r'charset=([\w-]+)')
+        charset = match1(
+            response.getheader('Content-Type'), r'charset=([\w-]+)'
+        )
         if charset is not None:
             data = data.decode(charset)
         else:
@@ -395,6 +426,7 @@ def get_content(url, headers={}, decoded=True):
 
     return data
 
+
 def post_content(url, headers={}, post_data={}, decoded=True):
     """Post the content of a URL via sending a HTTP POST request.
 
@@ -426,7 +458,9 @@ def post_content(url, headers={}, post_data={}, decoded=True):
 
     # Decode the response body
     if decoded:
-        charset = match1(response.getheader('Content-Type'), r'charset=([\w-]+)')
+        charset = match1(
+            response.getheader('Content-Type'), r'charset=([\w-]+)'
+        )
         if charset is not None:
             data = data.decode(charset)
         else:
@@ -434,21 +468,26 @@ def post_content(url, headers={}, post_data={}, decoded=True):
 
     return data
 
-def url_size(url, faker = False, headers = {}):
+
+def url_size(url, faker=False, headers={}):
     if faker:
-        response = urlopen_with_retry(request.Request(url, headers=fake_headers))
+        response = urlopen_with_retry(
+            request.Request(url, headers=fake_headers)
+        )
     elif headers:
         response = urlopen_with_retry(request.Request(url, headers=headers))
     else:
         response = urlopen_with_retry(url)
 
     size = response.headers['content-length']
-    return int(size) if size!=None else float('inf')
+    return int(size) if size is not None else float('inf')
 
-def urls_size(urls, faker = False, headers = {}):
+
+def urls_size(urls, faker=False, headers={}):
     return sum([url_size(url, faker=faker, headers=headers) for url in urls])
 
-def get_head(url, headers = {}, get_method = 'HEAD'):
+
+def get_head(url, headers={}, get_method='HEAD'):
     logging.debug('get_head: %s' % url)
 
     if headers:
@@ -459,11 +498,14 @@ def get_head(url, headers = {}, get_method = 'HEAD'):
     res = urlopen_with_retry(req)
     return dict(res.headers)
 
-def url_info(url, faker = False, headers = {}):
+
+def url_info(url, faker=False, headers={}):
     logging.debug('url_info: %s' % url)
 
     if faker:
-        response = urlopen_with_retry(request.Request(url, headers=fake_headers))
+        response = urlopen_with_retry(
+            request.Request(url, headers=fake_headers)
+        )
     elif headers:
         response = urlopen_with_retry(request.Request(url, headers=headers))
     else:
@@ -472,7 +514,8 @@ def url_info(url, faker = False, headers = {}):
     headers = response.headers
 
     type = headers['content-type']
-    if type == 'image/jpg; charset=UTF-8' or type == 'image/jpg' : type = 'audio/mpeg'    #fix for netease
+    if type == 'image/jpg; charset=UTF-8' or type == 'image/jpg':
+        type = 'audio/mpeg'  # fix for netease
     mapping = {
         'video/3gpp': '3gp',
         'video/f4v': 'flv',
@@ -498,7 +541,9 @@ def url_info(url, faker = False, headers = {}):
         type = None
         if headers['content-disposition']:
             try:
-                filename = parse.unquote(r1(r'filename="?([^"]+)"?', headers['content-disposition']))
+                filename = parse.unquote(
+                    r1(r'filename="?([^"]+)"?', headers['content-disposition'])
+                )
                 if len(filename.split('.')) > 1:
                     ext = filename.split('.')[-1]
                 else:
@@ -515,15 +560,20 @@ def url_info(url, faker = False, headers = {}):
 
     return type, ext, size
 
-def url_locations(urls, faker = False, headers = {}):
+
+def url_locations(urls, faker=False, headers={}):
     locations = []
     for url in urls:
         logging.debug('url_locations: %s' % url)
 
         if faker:
-            response = urlopen_with_retry(request.Request(url, headers=fake_headers))
+            response = urlopen_with_retry(
+                request.Request(url, headers=fake_headers)
+            )
         elif headers:
-            response = urlopen_with_retry(request.Request(url, headers=headers))
+            response = urlopen_with_retry(
+                request.Request(url, headers=headers)
+            )
         else:
             response = urlopen_with_retry(request.Request(url))
 
@@ -531,9 +581,13 @@ def url_locations(urls, faker = False, headers = {}):
     return locations
 
 
-def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers=None, timeout=None, **kwargs):
+def url_save(
+    url, filepath, bar, refer=None, is_part=False, faker=False,
+    headers=None, timeout=None, **kwargs
+):
     tmp_headers = headers.copy() if headers is not None else {}
-# When a referer specified with param refer, the key must be 'Referer' for the hack here
+    # When a referer specified with param refer,
+    # the key must be 'Referer' for the hack here
     if refer is not None:
         tmp_headers['Referer'] = refer
     file_size = url_size(url, faker=faker, headers=tmp_headers)
@@ -543,7 +597,11 @@ def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers
             if not is_part:
                 if bar:
                     bar.done()
-                print('Skipping %s: file already exists' % tr(os.path.basename(filepath)))
+                print(
+                    'Skipping {}: file already exists'.format(
+                        tr(os.path.basename(filepath))
+                    )
+                )
             else:
                 if bar:
                     bar.update_received(file_size)
@@ -556,7 +614,8 @@ def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers
     elif not os.path.exists(os.path.dirname(filepath)):
         os.mkdir(os.path.dirname(filepath))
 
-    temp_filepath = filepath + '.download' if file_size!=float('inf') else filepath
+    temp_filepath = filepath + '.download' if file_size != float('inf') \
+        else filepath
     received = 0
     if not force:
         open_mode = 'ab'
@@ -584,16 +643,27 @@ def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers
             tmp_headers['Referer'] = refer
 
         if timeout:
-            response = urlopen_with_retry(request.Request(url, headers=tmp_headers), timeout=timeout)
+            response = urlopen_with_retry(
+                request.Request(url, headers=tmp_headers), timeout=timeout
+            )
         else:
-            response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
+            response = urlopen_with_retry(
+                request.Request(url, headers=tmp_headers)
+            )
         try:
-            range_start = int(response.headers['content-range'][6:].split('/')[0].split('-')[0])
-            end_length = int(response.headers['content-range'][6:].split('/')[1])
+            range_start = int(
+                response.headers[
+                    'content-range'
+                ][6:].split('/')[0].split('-')[0]
+            )
+            end_length = int(
+                response.headers['content-range'][6:].split('/')[1]
+            )
             range_length = end_length - range_start
         except:
             content_length = response.headers['content-length']
-            range_length = int(content_length) if content_length!=None else float('inf')
+            range_length = int(content_length) if content_length is not None \
+                else float('inf')
 
         if file_size != received + range_length:
             received = 0
@@ -609,28 +679,33 @@ def url_save(url, filepath, bar, refer=None, is_part=False, faker=False, headers
                 except socket.timeout:
                     pass
                 if not buffer:
-                    if received == file_size: # Download finished
+                    if received == file_size:  # Download finished
                         break
                     # Unexpected termination. Retry request
                     tmp_headers['Range'] = 'bytes=' + str(received) + '-'
-                    response = urlopen_with_retry(request.Request(url, headers=tmp_headers))
+                    response = urlopen_with_retry(
+                        request.Request(url, headers=tmp_headers)
+                    )
                     continue
                 output.write(buffer)
                 received += len(buffer)
                 if bar:
                     bar.update_received(len(buffer))
 
-    assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (received, os.path.getsize(temp_filepath), temp_filepath)
+    assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (
+        received, os.path.getsize(temp_filepath), temp_filepath
+    )
 
     if os.access(filepath, os.W_OK):
-        os.remove(filepath) # on Windows rename could fail if destination filepath exists
+        # on Windows rename could fail if destination filepath exists
+        os.remove(filepath)
     os.rename(temp_filepath, filepath)
 
 
 class SimpleProgressBar:
     term_size = term.get_terminal_size()[1]
 
-    def __init__(self, total_size, total_pieces = 1):
+    def __init__(self, total_size, total_pieces=1):
         self.displayed = False
         self.total_size = total_size
         self.total_pieces = total_pieces
@@ -643,9 +718,12 @@ def __init__(self, total_size, total_pieces = 1):
         # 38 is the size of all statically known size in self.bar
         total_str = '%5s' % round(self.total_size / 1048576, 1)
         total_str_width = max(len(total_str), 5)
-        self.bar_size = self.term_size - 27 - 2*total_pieces_len - 2*total_str_width
+        self.bar_size = self.term_size - 27 - 2 * total_pieces_len \
+            - 2 * total_str_width
         self.bar = '{:>4}%% ({:>%s}/%sMB) ├{:─<%s}┤[{:>%s}/{:>%s}] {}' % (
-            total_str_width, total_str, self.bar_size, total_pieces_len, total_pieces_len)
+            total_str_width, total_str, self.bar_size, total_pieces_len,
+            total_pieces_len
+        )
 
     def update(self):
         self.displayed = True
@@ -662,7 +740,10 @@ def update(self):
         else:
             plus = ''
         bar = '█' * dots + plus
-        bar = self.bar.format(percent, round(self.received / 1048576, 1), bar, self.current_piece, self.total_pieces, self.speed)
+        bar = self.bar.format(
+            percent, round(self.received / 1048576, 1), bar,
+            self.current_piece, self.total_pieces, self.speed
+        )
         sys.stdout.write('\r' + bar)
         sys.stdout.flush()
 
@@ -689,8 +770,9 @@ def done(self):
             print()
             self.displayed = False
 
+
 class PiecesProgressBar:
-    def __init__(self, total_size, total_pieces = 1):
+    def __init__(self, total_size, total_pieces=1):
         self.displayed = False
         self.total_size = total_size
         self.total_pieces = total_pieces
@@ -699,7 +781,9 @@ def __init__(self, total_size, total_pieces = 1):
 
     def update(self):
         self.displayed = True
-        bar = '{0:>5}%[{1:<40}] {2}/{3}'.format('', '=' * 40, self.current_piece, self.total_pieces)
+        bar = '{0:>5}%[{1:<40}] {2}/{3}'.format(
+            '', '=' * 40, self.current_piece, self.total_pieces
+        )
         sys.stdout.write('\r' + bar)
         sys.stdout.flush()
 
@@ -715,16 +799,21 @@ def done(self):
             print()
             self.displayed = False
 
+
 class DummyProgressBar:
     def __init__(self, *args):
         pass
+
     def update_received(self, n):
         pass
+
     def update_piece(self, n):
         pass
+
     def done(self):
         pass
 
+
 def get_output_filename(urls, title, ext, output_dir, merge):
     # lame hack for the --output-filename option
     global output_filename
@@ -750,10 +839,17 @@ def get_output_filename(urls, title, ext, output_dir, merge):
                 merged_ext = 'ts'
     return '%s.%s' % (title, merged_ext)
 
-def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merge=True, faker=False, headers = {}, **kwargs):
+
+def download_urls(
+    urls, title, ext, total_size, output_dir='.', refer=None, merge=True,
+    faker=False, headers={}, **kwargs
+):
     assert urls
     if json_output:
-        json_output_.download_urls(urls=urls, title=title, ext=ext, total_size=total_size, refer=refer)
+        json_output_.download_urls(
+            urls=urls, title=title, ext=ext, total_size=total_size,
+            refer=refer
+        )
         return
     if dry_run:
         print('Real URLs:\n%s' % '\n'.join(urls))
@@ -776,7 +872,8 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
     output_filepath = os.path.join(output_dir, output_filename)
 
     if total_size:
-        if not force and os.path.exists(output_filepath) and os.path.getsize(output_filepath) >= total_size * 0.9:
+        if not force and os.path.exists(output_filepath) \
+                and os.path.getsize(output_filepath) >= total_size * 0.9:
             print('Skipping %s: file already exists' % output_filepath)
             print()
             return
@@ -788,7 +885,10 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
         url = urls[0]
         print('Downloading %s ...' % tr(output_filename))
         bar.update()
-        url_save(url, output_filepath, bar, refer = refer, faker = faker, headers = headers, **kwargs)
+        url_save(
+            url, output_filepath, bar, refer=refer, faker=faker,
+            headers=headers, **kwargs
+        )
         bar.done()
     else:
         parts = []
@@ -798,9 +898,12 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
             filename = '%s[%02d].%s' % (title, i, ext)
             filepath = os.path.join(output_dir, filename)
             parts.append(filepath)
-            #print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
+            # print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
             bar.update_piece(i + 1)
-            url_save(url, filepath, bar, refer = refer, is_part = True, faker = faker, headers = headers, **kwargs)
+            url_save(
+                url, filepath, bar, refer=refer, is_part=True, faker=faker,
+                headers=headers, **kwargs
+            )
         bar.done()
 
         if not merge:
@@ -814,7 +917,8 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 ret = ffmpeg_concat_av(parts, output_filepath, ext)
                 print('Merged into %s' % output_filename)
                 if ret == 0:
-                    for part in parts: os.remove(part)
+                    for part in parts:
+                        os.remove(part)
 
         elif ext in ['flv', 'f4v']:
             try:
@@ -848,7 +952,7 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
                 for part in parts:
                     os.remove(part)
 
-        elif ext == "ts":
+        elif ext == 'ts':
             try:
                 from .processor.ffmpeg import has_ffmpeg_installed
                 if has_ffmpeg_installed():
@@ -870,12 +974,15 @@ def download_urls(urls, title, ext, total_size, output_dir='.', refer=None, merg
     print()
 
 
-def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False):
+def download_rtmp_url(
+    url, title, ext, params={}, total_size=0, output_dir='.', refer=None,
+    merge=True, faker=False
+):
     assert url
     if dry_run:
         print('Real URL:\n%s\n' % [url])
-        if params.get("-y",False): #None or unset ->False
-            print('Real Playpath:\n%s\n' % [params.get("-y")])
+        if params.get('-y', False):  # None or unset -> False
+            print('Real Playpath:\n%s\n' % [params.get('-y')])
         return
 
     if player:
@@ -883,16 +990,22 @@ def download_rtmp_url(url,title, ext,params={}, total_size=0, output_dir='.', re
         play_rtmpdump_stream(player, url, params)
         return
 
-    from .processor.rtmpdump import has_rtmpdump_installed, download_rtmpdump_stream
-    assert has_rtmpdump_installed(), "RTMPDump not installed."
-    download_rtmpdump_stream(url,  title, ext,params, output_dir)
+    from .processor.rtmpdump import (
+        has_rtmpdump_installed, download_rtmpdump_stream
+    )
+    assert has_rtmpdump_installed(), 'RTMPDump not installed.'
+    download_rtmpdump_stream(url,  title, ext, params, output_dir)
+
 
-def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.', refer=None, merge=True, faker=False, stream=True):
+def download_url_ffmpeg(
+    url, title, ext, params={}, total_size=0, output_dir='.', refer=None,
+    merge=True, faker=False, stream=True
+):
     assert url
     if dry_run:
         print('Real URL:\n%s\n' % [url])
-        if params.get("-y",False): #None or unset ->False
-            print('Real Playpath:\n%s\n' % [params.get("-y")])
+        if params.get('-y', False):  # None or unset ->False
+            print('Real Playpath:\n%s\n' % [params.get('-y')])
         return
 
     if player:
@@ -900,11 +1013,11 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
         return
 
     from .processor.ffmpeg import has_ffmpeg_installed, ffmpeg_download_stream
-    assert has_ffmpeg_installed(), "FFmpeg not installed."
+    assert has_ffmpeg_installed(), 'FFmpeg not installed.'
 
     global output_filename
     if output_filename:
-        dotPos = output_filename.rfind(".")
+        dotPos = output_filename.rfind('.')
         title = output_filename[:dotPos]
         ext = output_filename[dotPos+1:]
 
@@ -912,14 +1025,18 @@ def download_url_ffmpeg(url,title, ext,params={}, total_size=0, output_dir='.',
 
     ffmpeg_download_stream(url, title, ext, params, output_dir, stream=stream)
 
+
 def playlist_not_supported(name):
     def f(*args, **kwargs):
         raise NotImplementedError('Playlist is not supported for ' + name)
     return f
 
+
 def print_info(site_info, title, type, size, **kwargs):
     if json_output:
-        json_output_.print_info(site_info=site_info, title=title, type=type, size=size)
+        json_output_.print_info(
+            site_info=site_info, title=title, type=type, size=size
+        )
         return
     if type:
         type = type.lower()
@@ -950,40 +1067,40 @@ def print_info(site_info, title, type, size, **kwargs):
         type = 'image/gif'
 
     if type in ['video/3gpp']:
-        type_info = "3GPP multimedia file (%s)" % type
+        type_info = '3GPP multimedia file (%s)' % type
     elif type in ['video/x-flv', 'video/f4v']:
-        type_info = "Flash video (%s)" % type
+        type_info = 'Flash video (%s)' % type
     elif type in ['video/mp4', 'video/x-m4v']:
-        type_info = "MPEG-4 video (%s)" % type
+        type_info = 'MPEG-4 video (%s)' % type
     elif type in ['video/MP2T']:
-        type_info = "MPEG-2 transport stream (%s)" % type
+        type_info = 'MPEG-2 transport stream (%s)' % type
     elif type in ['video/webm']:
-        type_info = "WebM video (%s)" % type
-    #elif type in ['video/ogg']:
-    #    type_info = "Ogg video (%s)" % type
+        type_info = 'WebM video (%s)' % type
+    # elif type in ['video/ogg']:
+    #    type_info = 'Ogg video (%s)' % type
     elif type in ['video/quicktime']:
-        type_info = "QuickTime video (%s)" % type
+        type_info = 'QuickTime video (%s)' % type
     elif type in ['video/x-matroska']:
-        type_info = "Matroska video (%s)" % type
-    #elif type in ['video/x-ms-wmv']:
-    #    type_info = "Windows Media video (%s)" % type
+        type_info = 'Matroska video (%s)' % type
+    # elif type in ['video/x-ms-wmv']:
+    #    type_info = 'Windows Media video (%s)' % type
     elif type in ['video/x-ms-asf']:
-        type_info = "Advanced Systems Format (%s)" % type
-    #elif type in ['video/mpeg']:
-    #    type_info = "MPEG video (%s)" % type
+        type_info = 'Advanced Systems Format (%s)' % type
+    # elif type in ['video/mpeg']:
+    #    type_info = 'MPEG video (%s)' % type
     elif type in ['audio/mp4', 'audio/m4a']:
-        type_info = "MPEG-4 audio (%s)" % type
+        type_info = 'MPEG-4 audio (%s)' % type
     elif type in ['audio/mpeg']:
-        type_info = "MP3 (%s)" % type
+        type_info = 'MP3 (%s)' % type
     elif type in ['audio/wav', 'audio/wave', 'audio/x-wav']:
         type_info = 'Waveform Audio File Format ({})'.format(type)
 
     elif type in ['image/jpeg']:
-        type_info = "JPEG Image (%s)" % type
+        type_info = 'JPEG Image (%s)' % type
     elif type in ['image/png']:
-        type_info = "Portable Network Graphics (%s)" % type
+        type_info = 'Portable Network Graphics (%s)' % type
     elif type in ['image/gif']:
-        type_info = "Graphics Interchange Format (%s)" % type
+        type_info = 'Graphics Interchange Format (%s)' % type
     elif type in ['m3u8']:
         if 'm3u8_type' in kwargs:
             if kwargs['m3u8_type'] == 'master':
@@ -991,17 +1108,21 @@ def print_info(site_info, title, type, size, **kwargs):
         else:
             type_info = 'M3U8 Playlist {}'.format(type)
     else:
-        type_info = "Unknown type (%s)" % type
+        type_info = 'Unknown type (%s)' % type
 
-    maybe_print("Site:      ", site_info)
-    maybe_print("Title:     ", unescape_html(tr(title)))
-    print("Type:      ", type_info)
+    maybe_print('Site:      ', site_info)
+    maybe_print('Title:     ', unescape_html(tr(title)))
+    print('Type:      ', type_info)
     if type != 'm3u8':
-        print("Size:      ", round(size / 1048576, 2), "MiB (" + str(size) + " Bytes)")
+        print(
+            'Size:      ', round(size / 1048576, 2),
+            'MiB (' + str(size) + ' Bytes)'
+        )
     if type == 'm3u8' and 'm3u8_url' in kwargs:
         print('M3U8 Url:   {}'.format(kwargs['m3u8_url']))
     print()
 
+
 def mime_to_container(mime):
     mapping = {
         'video/3gpp': '3gp',
@@ -1014,6 +1135,7 @@ def mime_to_container(mime):
     else:
         return mime.split('/')[1]
 
+
 def parse_host(host):
     """Parses host name and port number from a string.
     """
@@ -1026,6 +1148,7 @@ def parse_host(host):
     port = o.port or 0
     return (hostname, port)
 
+
 def set_proxy(proxy):
     proxy_handler = request.ProxyHandler({
         'http': '%s:%s' % proxy,
@@ -1034,27 +1157,33 @@ def set_proxy(proxy):
     opener = request.build_opener(proxy_handler)
     request.install_opener(opener)
 
+
 def unset_proxy():
     proxy_handler = request.ProxyHandler({})
     opener = request.build_opener(proxy_handler)
     request.install_opener(opener)
 
+
 # DEPRECATED in favor of set_proxy() and unset_proxy()
 def set_http_proxy(proxy):
-    if proxy == None: # Use system default setting
+    if proxy is None:  # Use system default setting
         proxy_support = request.ProxyHandler()
-    elif proxy == '': # Don't use any proxy
+    elif proxy == '':  # Don't use any proxy
         proxy_support = request.ProxyHandler({})
-    else: # Use proxy
-        proxy_support = request.ProxyHandler({'http': '%s' % proxy, 'https': '%s' % proxy})
+    else:  # Use proxy
+        proxy_support = request.ProxyHandler(
+            {'http': '%s' % proxy, 'https': '%s' % proxy}
+        )
     opener = request.build_opener(proxy_support)
     request.install_opener(opener)
 
+
 def print_more_compatible(*args, **kwargs):
     import builtins as __builtin__
     """Overload default print function as py (<3.3) does not support 'flush' keyword.
     Although the function name can be same as print to get itself overloaded automatically,
-    I'd rather leave it with a different name and only overload it when importing to make less confusion. """
+    I'd rather leave it with a different name and only overload it when importing to make less confusion.
+    """
     # nothing happens on py3.3 and later
     if sys.version_info[:2] >= (3, 3):
         return __builtin__.print(*args, **kwargs)
@@ -1077,6 +1206,7 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         else:
             download(url, **kwargs)
 
+
 def load_cookies(cookiefile):
     global cookies
     try:
@@ -1094,7 +1224,7 @@ def load_cookies(cookiefile):
                 c = cookiejar.Cookie(
                     0, item[4], item[5], None, False, item[0],
                     item[0].startswith('.'), item[0].startswith('.'),
-                    item[1], False, item[2], item[3], item[3]=="", None,
+                    item[1], False, item[2], item[3], item[3] == '', None,
                     None, {},
                 )
                 cookies.set_cookie(c)
@@ -1105,28 +1235,42 @@ def load_cookies(cookiefile):
         # FROM cookies
         # http://n8henrie.com/2013/11/use-chromes-cookies-for-easier-downloading-with-python-requests/
 
+
 def set_socks_proxy(proxy):
     try:
         import socks
         socks_proxy_addrs = proxy.split(':')
-        socks.set_default_proxy(socks.SOCKS5,
-                                socks_proxy_addrs[0],
-                                int(socks_proxy_addrs[1]))
+        socks.set_default_proxy(
+            socks.SOCKS5,
+            socks_proxy_addrs[0],
+            int(socks_proxy_addrs[1])
+        )
         socket.socket = socks.socksocket
+
         def getaddrinfo(*args):
-            return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))]
+            return [
+                (socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))
+            ]
         socket.getaddrinfo = getaddrinfo
     except ImportError:
-        log.w('Error importing PySocks library, socks proxy ignored.'
-            'In order to use use socks proxy, please install PySocks.')
+        log.w(
+            'Error importing PySocks library, socks proxy ignored.'
+            'In order to use use socks proxy, please install PySocks.'
+        )
+
 
 def script_main(download, download_playlist, **kwargs):
     logging.basicConfig(format='[%(levelname)s] %(message)s')
 
     def print_version():
-        log.i('version %s, a tiny downloader that scrapes the web.'
-                % get_version(kwargs['repo_path']
-            if 'repo_path' in kwargs else __version__))
+        version = get_version(
+            kwargs['repo_path'] if 'repo_path' in kwargs else __version__
+        )
+        log.i(
+            'version {}, a tiny downloader that scrapes the web.'.format(
+                version
+            )
+        )
 
     parser = argparse.ArgumentParser(
         prog='you-get',
@@ -1134,66 +1278,105 @@ def print_version():
         description='A tiny downloader that scrapes the web',
         add_help=False,
     )
-    parser.add_argument('-V', '--version', action='store_true',
-                        help='Print version and exit')
-    parser.add_argument('-h', '--help', action='store_true',
-                        help='Print this help message and exit')
+    parser.add_argument(
+        '-V', '--version', action='store_true',
+        help='Print version and exit'
+    )
+    parser.add_argument(
+        '-h', '--help', action='store_true',
+        help='Print this help message and exit'
+    )
 
-    dry_run_grp = parser.add_argument_group('Dry-run options', '(no actual downloading)')
+    dry_run_grp = parser.add_argument_group(
+        'Dry-run options', '(no actual downloading)'
+    )
     dry_run_grp = dry_run_grp.add_mutually_exclusive_group()
-    dry_run_grp.add_argument('-i', '--info', action='store_true',
-                        help='Print extracted information')
-    dry_run_grp.add_argument('-u', '--url', action='store_true',
-                        help='Print extracted information with URLs')
-    dry_run_grp.add_argument('--json', action='store_true',
-                        help='Print extracted URLs in JSON format')
+    dry_run_grp.add_argument(
+        '-i', '--info', action='store_true', help='Print extracted information'
+    )
+    dry_run_grp.add_argument(
+        '-u', '--url', action='store_true',
+        help='Print extracted information with URLs'
+    )
+    dry_run_grp.add_argument(
+        '--json', action='store_true',
+        help='Print extracted URLs in JSON format'
+    )
 
     download_grp = parser.add_argument_group('Download options')
-    download_grp.add_argument('-n', '--no-merge', action='store_true', default=False,
-                        help='Do not merge video parts')
-    download_grp.add_argument('--no-caption', action='store_true',
-                        help='Do not download captions (subtitles, lyrics, danmaku, ...)')
-    download_grp.add_argument('-f', '--force', action='store_true', default=False,
-                        help='Force overwriting existing files')
-    download_grp.add_argument('-F', '--format', metavar='STREAM_ID',
-                        help='Set video format to STREAM_ID')
-    download_grp.add_argument('-O', '--output-filename', metavar='FILE',
-                        help='Set output filename')
-    download_grp.add_argument('-o', '--output-dir', metavar='DIR', default='.',
-                        help='Set output directory')
-    download_grp.add_argument('-p', '--player', metavar='PLAYER',
-                        help='Stream extracted URL to a PLAYER')
-    download_grp.add_argument('-c', '--cookies', metavar='COOKIES_FILE',
-                        help='Load cookies.txt or cookies.sqlite')
-    download_grp.add_argument('-t', '--timeout', metavar='SECONDS', type=int, default=600,
-                        help='Set socket timeout')
-    download_grp.add_argument('-d', '--debug', action='store_true',
-                        help='Show traceback and other debug info')
-    download_grp.add_argument('-I', '--input-file', metavar='FILE', type=argparse.FileType('r'),
-                        help='Read non-playlist URLs from FILE')
-    download_grp.add_argument('-P', '--password',
-                        help='Set video visit password to PASSWORD')
-    download_grp.add_argument('-l', '--playlist', action='store_true',
-                        help='Prefer to download a playlist')
+    download_grp.add_argument(
+        '-n', '--no-merge', action='store_true', default=False,
+        help='Do not merge video parts'
+    )
+    download_grp.add_argument(
+        '--no-caption', action='store_true',
+        help='Do not download captions (subtitles, lyrics, danmaku, ...)'
+    )
+    download_grp.add_argument(
+        '-f', '--force', action='store_true', default=False,
+        help='Force overwriting existing files'
+    )
+    download_grp.add_argument(
+        '-F', '--format', metavar='STREAM_ID',
+        help='Set video format to STREAM_ID'
+    )
+    download_grp.add_argument(
+        '-O', '--output-filename', metavar='FILE', help='Set output filename'
+    )
+    download_grp.add_argument(
+        '-o', '--output-dir', metavar='DIR', default='.',
+        help='Set output directory'
+    )
+    download_grp.add_argument(
+        '-p', '--player', metavar='PLAYER',
+        help='Stream extracted URL to a PLAYER'
+    )
+    download_grp.add_argument(
+        '-c', '--cookies', metavar='COOKIES_FILE',
+        help='Load cookies.txt or cookies.sqlite'
+    )
+    download_grp.add_argument(
+        '-t', '--timeout', metavar='SECONDS', type=int, default=600,
+        help='Set socket timeout'
+    )
+    download_grp.add_argument(
+        '-d', '--debug', action='store_true',
+        help='Show traceback and other debug info'
+    )
+    download_grp.add_argument(
+        '-I', '--input-file', metavar='FILE', type=argparse.FileType('r'),
+        help='Read non-playlist URLs from FILE'
+    )
+    download_grp.add_argument(
+        '-P', '--password', help='Set video visit password to PASSWORD'
+    )
+    download_grp.add_argument(
+        '-l', '--playlist', action='store_true',
+        help='Prefer to download a playlist'
+    )
 
     proxy_grp = parser.add_argument_group('Proxy options')
     proxy_grp = proxy_grp.add_mutually_exclusive_group()
-    proxy_grp.add_argument('-x', '--http-proxy', metavar='HOST:PORT',
-                        help='Use an HTTP proxy for downloading')
-    proxy_grp.add_argument('-y', '--extractor-proxy', metavar='HOST:PORT',
-                        help='Use an HTTP proxy for extracting only')
-    proxy_grp.add_argument('--no-proxy', action='store_true',
-                        help='Never use a proxy')
-    proxy_grp.add_argument('-s', '--socks-proxy', metavar='HOST:PORT',
-                        help='Use an SOCKS5 proxy for downloading')
-
-    download_grp.add_argument('--stream',
-                        help=argparse.SUPPRESS)
-    download_grp.add_argument('--itag',
-                        help=argparse.SUPPRESS)
-
-    parser.add_argument('URL', nargs='*',
-                        help=argparse.SUPPRESS)
+    proxy_grp.add_argument(
+        '-x', '--http-proxy', metavar='HOST:PORT',
+        help='Use an HTTP proxy for downloading'
+    )
+    proxy_grp.add_argument(
+        '-y', '--extractor-proxy', metavar='HOST:PORT',
+        help='Use an HTTP proxy for extracting only'
+    )
+    proxy_grp.add_argument(
+        '--no-proxy', action='store_true', help='Never use a proxy'
+    )
+    proxy_grp.add_argument(
+        '-s', '--socks-proxy', metavar='HOST:PORT',
+        help='Use an SOCKS5 proxy for downloading'
+    )
+
+    download_grp.add_argument('--stream', help=argparse.SUPPRESS)
+    download_grp.add_argument('--itag', help=argparse.SUPPRESS)
+
+    parser.add_argument('URL', nargs='*', help=argparse.SUPPRESS)
 
     args = parser.parse_args()
 
@@ -1250,7 +1433,10 @@ def print_version():
     if args.input_file:
         logging.debug('you are trying to load urls from %s', args.input_file)
         if args.playlist:
-            log.e("reading playlist from a file is unsupported and won't make your life easier")
+            log.e(
+                "reading playlist from a file is unsupported "
+                "and won't make your life easier"
+            )
             sys.exit(2)
         URLs.extend(args.input_file.read().splitlines())
         args.input_file.close()
@@ -1272,7 +1458,8 @@ def print_version():
             download, download_playlist,
             URLs, args.playlist,
             output_dir=args.output_dir, merge=not args.no_merge,
-            info_only=info_only, json_output=json_output, caption=caption, password=args.password,
+            info_only=info_only, json_output=json_output, caption=caption,
+            password=args.password,
             **extra
         )
     except KeyboardInterrupt:
@@ -1283,9 +1470,14 @@ def print_version():
     except UnicodeEncodeError:
         if args.debug:
             raise
-        log.e('[error] oops, the current environment does not seem to support Unicode.')
+        log.e(
+            '[error] oops, the current environment does not seem to support '
+            'Unicode.'
+        )
         log.e('please set it to a UTF-8-aware locale first,')
-        log.e('so as to save the video (with some Unicode characters) correctly.')
+        log.e(
+            'so as to save the video (with some Unicode characters) correctly.'
+        )
         log.e('you can do it like this:')
         log.e('    (Windows)    % chcp 65001 ')
         log.e('    (Linux)      $ LC_CTYPE=en_US.UTF-8')
@@ -1293,7 +1485,9 @@ def print_version():
     except Exception:
         if not args.debug:
             log.e('[error] oops, something went wrong.')
-            log.e('don\'t panic, c\'est la vie. please try the following steps:')
+            log.e(
+                'don\'t panic, c\'est la vie. please try the following steps:'
+            )
             log.e('  (1) Rule out any network problem.')
             log.e('  (2) Make sure you-get is up-to-date.')
             log.e('  (3) Check if the issue is already known, on')
@@ -1307,22 +1501,28 @@ def print_version():
             raise
         sys.exit(1)
 
+
 def google_search(url):
     keywords = r1(r'https?://(.*)', url)
     url = 'https://www.google.com/search?tbm=vid&q=%s' % parse.quote(keywords)
     page = get_content(url, headers=fake_headers)
-    videos = re.findall(r'<a href="(https?://[^"]+)" onmousedown="[^"]+">([^<]+)<', page)
+    videos = re.findall(
+        r'<a href="(https?://[^"]+)" onmousedown="[^"]+">([^<]+)<', page
+    )
     vdurs = re.findall(r'<span class="vdur _dwc">([^<]+)<', page)
     durs = [r1(r'(\d+:\d+)', unescape_html(dur)) for dur in vdurs]
-    print("Google Videos search:")
+    print('Google Videos search:')
     for v in zip(videos, durs):
-        print("- video:  %s [%s]" % (unescape_html(v[0][1]),
-                                     v[1] if v[1] else '?'))
-        print("# you-get %s" % log.sprint(v[0][0], log.UNDERLINE))
+        print('- video:  {} [{}]'.format(
+            unescape_html(v[0][1]),
+            v[1] if v[1] else '?'
+        ))
+        print('# you-get %s' % log.sprint(v[0][0], log.UNDERLINE))
         print()
-    print("Best matched result:")
+    print('Best matched result:')
     return(videos[0][0])
 
+
 def url_to_module(url):
     try:
         video_host = r1(r'https?://([^/]+)/', url)
@@ -1340,15 +1540,18 @@ def url_to_module(url):
 
     k = r1(r'([^.]+)', domain)
     if k in SITES:
-        return import_module('.'.join(['you_get', 'extractors', SITES[k]])), url
+        return (
+            import_module('.'.join(['you_get', 'extractors', SITES[k]])),
+            url
+        )
     else:
         import http.client
-        video_host = r1(r'https?://([^/]+)/', url) # .cn could be removed
+        video_host = r1(r'https?://([^/]+)/', url)  # .cn could be removed
         if url.startswith('https://'):
             conn = http.client.HTTPSConnection(video_host)
         else:
             conn = http.client.HTTPConnection(video_host)
-        conn.request("HEAD", video_url, headers=fake_headers)
+        conn.request('HEAD', video_url, headers=fake_headers)
         res = conn.getresponse()
         location = res.getheader('location')
         if location and location != url and not location.startswith('/'):
@@ -1356,13 +1559,16 @@ def url_to_module(url):
         else:
             return import_module('you_get.extractors.universal'), url
 
+
 def any_download(url, **kwargs):
     m, url = url_to_module(url)
     m.download(url, **kwargs)
 
+
 def any_download_playlist(url, **kwargs):
     m, url = url_to_module(url)
     m.download_playlist(url, **kwargs)
 
+
 def main(**kwargs):
     script_main(any_download, any_download_playlist, **kwargs)

From 99a51012f0726064714a564d34d6db2e1f49edaf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 17 Nov 2017 17:08:31 +0100
Subject: [PATCH 0552/1225] [util] trim title length <= 80 (follow-up of
 ed12df2)

---
 src/you_get/util/fs.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
index 4f415bf070..d49a117d62 100644
--- a/src/you_get/util/fs.py
+++ b/src/you_get/util/fs.py
@@ -41,5 +41,5 @@ def legitimize(text, os=platform.system()):
         if text.startswith("."):
             text = text[1:]
 
-    text = text[:82] # Trim to 82 Unicode characters long
+    text = text[:80] # Trim to 82 Unicode characters long
     return text

From 2edb539d1fd85d970be3ea897766d82e48df8f06 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 17 Nov 2017 17:14:47 +0100
Subject: [PATCH 0553/1225] version 0.4.964

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 63d93414ab..88315252c3 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.939'
+__version__ = '0.4.964'

From 44974ade13ec49311aceb3107ae99f92ae784801 Mon Sep 17 00:00:00 2001
From: Sergey Zmushko <zmuxsm@gmail.com>
Date: Sun, 19 Nov 2017 00:53:03 +0300
Subject: [PATCH 0554/1225] [vine] fix empty file name if no description found

Now if no description in video file metadata found then user name and
video id will be used
---
 src/you_get/extractors/vine.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/vine.py b/src/you_get/extractors/vine.py
index 491ced73ff..d75454cf54 100644
--- a/src/you_get/extractors/vine.py
+++ b/src/you_get/extractors/vine.py
@@ -5,24 +5,24 @@
 from ..common import *
 import json
 
+
 def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html(url)
+    html = get_content(url)
 
-    vid = r1(r'vine.co/v/([^/]+)', url)
+    video_id = r1(r'vine.co/v/([^/]+)', url)
     title = r1(r'<title>([^<]*)</title>', html)
     stream = r1(r'<meta property="twitter:player:stream" content="([^"]*)">', html)
-    if not stream: # https://vine.co/v/.../card
+    if not stream:  # https://vine.co/v/.../card
         stream = r1(r'"videoUrl":"([^"]+)"', html)
         if stream:
             stream = stream.replace('\\/', '/')
         else:
-            if url[-1] == '/':
-                url = url[:-1]
-            video_id = url.split('/')[-1]
             posts_url = 'https://archive.vine.co/posts/' + video_id + '.json'
             json_data = json.loads(get_content(posts_url))
             stream = json_data['videoDashUrl']
             title = json_data['description']
+            if title == "":
+                title = json_data['username'].replace(" ", "_") + "_" + video_id
 
     mime, ext, size = url_info(stream)
 
@@ -30,6 +30,7 @@ def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if not info_only:
         download_urls([stream], title, ext, size, output_dir, merge=merge)
 
+
 site_info = "Vine.co"
 download = vine_download
 download_playlist = playlist_not_supported('vine')

From 917c2cc1c4df049e0802316a32cb579212190fce Mon Sep 17 00:00:00 2001
From: Chinsyo <chinsyo@sina.cn>
Date: Mon, 20 Nov 2017 12:45:35 +0800
Subject: [PATCH 0555/1225] Update the regex for mgtv extractor, adapt to https

---
 src/you_get/extractors/mgtv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 8337b88747..730dfeba44 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -27,9 +27,9 @@ class MGTV(VideoExtractor):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        vid = match1(url, 'http://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
+        vid = match1(url, 'https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
         if not vid:
-            vid = match1(url, 'http://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
+            vid = match1(url, 'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
         return vid
     
     #----------------------------------------------------------------------

From b787226053fc6a7a267227fda8fe668d96f8658b Mon Sep 17 00:00:00 2001
From: Zhiming Wang <zmwangx@gmail.com>
Date: Wed, 22 Nov 2017 09:13:17 -0500
Subject: [PATCH 0556/1225] [bilibili] fix livestream extractor

The HTML page of a live page does not carry any room info now. Room ID
and title needs to be retrieved with proper API calls.

Fixes #2479.
---
 src/you_get/extractors/bilibili.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)
 mode change 100644 => 100755 src/you_get/extractors/bilibili.py

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
old mode 100644
new mode 100755
index f75397d6a7..3b5aabea3b
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -25,6 +25,8 @@ class Bilibili(VideoExtractor):
     live_api = 'http://live.bilibili.com/api/playurl?cid={}&otype=json'
     api_url = 'http://interface.bilibili.com/playurl?'
     bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
+    live_room_init_api_url = 'https://api.live.bilibili.com/room/v1/Room/room_init?id={}'
+    live_room_info_api_url = 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id={}'
 
     SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
     SEC2 = '9b288147e5474dd2aa67085f716c560d'
@@ -188,8 +190,17 @@ def entry(self, **kwargs):
             return
 
     def live_entry(self, **kwargs):
-        self.title = re.search(r'<title>([^<]+)', self.page).group(1)
-        self.room_id = re.search('ROOMID\s*=\s*(\d+)', self.page).group(1)
+        # Extract room ID from the short display ID (seen in the room
+        # URL). The room ID is usually the same as the short ID, but not
+        # always; case in point: https://live.bilibili.com/48, with 48
+        # as the short ID and 63727 as the actual ID.
+        room_short_id = re.search(r'live.bilibili.com/([^?]+)', self.url).group(1)
+        room_init_api_response = json.loads(get_content(self.live_room_init_api_url.format(room_short_id)))
+        self.room_id = room_init_api_response['data']['room_id']
+
+        room_info_api_response = json.loads(get_content(self.live_room_info_api_url.format(self.room_id)))
+        self.title = room_info_api_response['data']['title']
+
         api_url = self.live_api.format(self.room_id)
         json_data = json.loads(get_content(api_url))
         urls = [json_data['durl'][0]['url']]

From 4f7ba5aac8965b55fa0733f74d9a7cfb5a414ae3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 25 Nov 2017 13:21:49 +0100
Subject: [PATCH 0557/1225] [youku] update ccode

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 65a60f580f..cd4483a1ad 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0401'
+        self.ccode = '0406'
         self.utid = None
 
     def youku_ups(self):
@@ -154,7 +154,7 @@ def prepare(self, **kwargs):
                     log.wtf('Cannot fetch vid')
 
         if kwargs.get('src') and kwargs['src'] == 'tudou':
-            self.ccode = '050F'
+            self.ccode = '0406'
 
         if kwargs.get('password') and kwargs['password']:
             self.password_protected = True

From f3290c717dcd942814b5881ed6db409b693b3863 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 25 Nov 2017 13:54:50 +0100
Subject: [PATCH 0558/1225] [bilibili] mode change 100755 => 100644

---
 src/you_get/extractors/bilibili.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/you_get/extractors/bilibili.py

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
old mode 100755
new mode 100644

From b2d97bcaea247e1aabfb2ced7d326866c127c6fb Mon Sep 17 00:00:00 2001
From: Sergey Zmushko <zmuxsm@gmail.com>
Date: Sun, 19 Nov 2017 21:07:51 +0300
Subject: [PATCH 0559/1225] [coub] add coub.com support

Video and audio files with high quality are downloaded.
We use ffmpeg for audio and video combining into one file
---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/coub.py     | 91 ++++++++++++++++++++++++++++++
 src/you_get/processor/ffmpeg.py    | 31 +++++++++-
 4 files changed, 122 insertions(+), 2 deletions(-)
 create mode 100644 src/you_get/extractors/coub.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index fe8fbbd028..ce7ee11e03 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -13,6 +13,7 @@
     'cctv'             : 'cntv',
     'cntv'             : 'cntv',
     'cbs'              : 'cbs',
+    'coub'             : 'coub',
     'dailymotion'      : 'dailymotion',
     'dilidili'         : 'dilidili',
     'douban'           : 'douban',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index b078db0876..f1acf00039 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -11,6 +11,7 @@
 from .cbs import *
 from .ckplayer import *
 from .cntv import *
+from .coub import *
 from .dailymotion import *
 from .dilidili import *
 from .douban import *
diff --git a/src/you_get/extractors/coub.py b/src/you_get/extractors/coub.py
new file mode 100644
index 0000000000..7f5dccb665
--- /dev/null
+++ b/src/you_get/extractors/coub.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+
+__all__ = ['coub_download']
+
+from ..common import *
+from ..processor import ffmpeg
+from ..util.fs import legitimize
+
+
+def coub_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    html = get_content(url)
+
+    try:
+        json_data = get_coub_data(html)
+        title, video_url, audio_url = get_title_and_urls(json_data)
+        video_file_name, video_file_path = get_file_path(merge, output_dir, title, video_url)
+        audio_file_name, audio_file_path = get_file_path(merge, output_dir, title, audio_url)
+        download_url(audio_url, merge, output_dir, title, info_only)
+        download_url(video_url, merge, output_dir, title, info_only)
+        if not info_only:
+            try:
+                fix_coub_video_file(video_file_path)
+                audio_duration = float(ffmpeg.ffprobe_get_media_duration(audio_file_path))
+                video_duration = float(ffmpeg.ffprobe_get_media_duration(video_file_path))
+                loop_file_path = get_loop_file_path(title, output_dir)
+                single_file_path = audio_file_path
+                if audio_duration > video_duration:
+                    write_loop_file(int(audio_duration / video_duration), loop_file_path, video_file_name)
+                else:
+                    single_file_path = audio_file_path
+                    write_loop_file(int(video_duration / audio_duration), loop_file_path, audio_file_name)
+
+                ffmpeg.ffmpeg_concat_audio_and_video([loop_file_path, single_file_path], title + "_full", "mp4")
+                cleanup_files([video_file_path, audio_file_path, loop_file_path])
+            except EnvironmentError as err:
+                print("Error preparing full coub video. {}".format(err))
+    except Exception as err:
+        print("Error while downloading files. {}".format(err))
+
+
+def write_loop_file(records_number, loop_file_path, file_name):
+    with open(loop_file_path, 'a') as file:
+        for i in range(records_number):
+            file.write("file '{}'\n".format(file_name))
+
+
+def download_url(url, merge, output_dir, title, info_only):
+    mime, ext, size = url_info(url)
+    print_info(site_info, title, mime, size)
+    if not info_only:
+        download_urls([url], title, ext, size, output_dir, merge=merge)
+
+
+def fix_coub_video_file(file_path):
+    with open(file_path, 'r+b') as file:
+        file.seek(0)
+        file.write(bytes(2))
+
+
+def get_title_and_urls(json_data):
+    title = legitimize(json_data['title'].replace(" ", "_"))
+    video_url = json_data['file_versions']['html5']['video']['high']['url']
+    audio_url = json_data['file_versions']['html5']['audio']['high']['url']
+    return title, video_url, audio_url
+
+
+def get_coub_data(html):
+    coub_data = r1(r'<script id=\'coubPageCoubJson\' type=\'text/json\'>([^<]+)</script>', html)
+    json_data = json.loads(coub_data)
+    return json_data
+
+
+def get_file_path(merge, output_dir, title, url):
+    mime, ext, size = url_info(url)
+    file_name = get_output_filename([], title, ext, output_dir, merge)
+    file_path = os.path.join(output_dir, file_name)
+    return file_name, file_path
+
+
+def get_loop_file_path(title, output_dir):
+    return os.path.join(output_dir, get_output_filename([], title, "txt", None, False))
+
+
+def cleanup_files(files):
+    for file in files:
+        os.remove(file)
+
+
+site_info = "coub.com"
+download = coub_download
+download_playlist = playlist_not_supported('coub')
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index c7b362e103..89d53e5009 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -28,11 +28,11 @@ def get_usable_ffmpeg(cmd):
             print('It seems that your ffmpeg is a nightly build.')
             print('Please switch to the latest stable if merging failed.')
             version = [1, 0]
-        return cmd, version
+        return cmd, 'ffprobe', version
     except:
         return None
 
-FFMPEG, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None)
+FFMPEG, FFPROBE, FFMPEG_VERSION = get_usable_ffmpeg('ffmpeg') or get_usable_ffmpeg('avconv') or (None, None, None)
 if logging.getLogger().isEnabledFor(logging.DEBUG):
     LOGLEVEL = ['-loglevel', 'info']
     STDIN = None
@@ -250,3 +250,30 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=
             pass
 
     return True
+
+
+def ffmpeg_concat_audio_and_video(files, output, ext):
+    print('Merging video and audio parts... ', end="", flush=True)
+    if has_ffmpeg_installed:
+        params = [FFMPEG] + LOGLEVEL
+        params.extend(['-f', 'concat'])
+        for file in files:
+            if os.path.isfile(file):
+                params.extend(['-i', file])
+        params.extend(['-c:v', 'copy'])
+        params.extend(['-c:a', 'aac'])
+        params.extend(['-strict', 'experimental'])
+        params.append(output+"."+ext)
+        return subprocess.call(params, stdin=STDIN)
+    else:
+        raise EnvironmentError('No ffmpeg found')
+
+
+def ffprobe_get_media_duration(file):
+    print('Getting {} duration'.format(file))
+    params = [FFPROBE]
+    params.extend(['-i', file])
+    params.extend(['-show_entries', 'format=duration'])
+    params.extend(['-v', 'quiet'])
+    params.extend(['-of', 'csv=p=0'])
+    return subprocess.check_output(params, stdin=STDIN, stderr=subprocess.STDOUT).decode().strip()

From 33664c514c0572106e76739e473c92cf3804c7a6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 21:09:43 +0100
Subject: [PATCH 0560/1225] [common] formatting

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f2a6923828..5c69f03016 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -120,7 +120,7 @@
     'youtu'            : 'youtube',
     'youtube'          : 'youtube',
     'zhanqi'           : 'zhanqi',
-    '365yg': 'toutiao',
+    '365yg'            : 'toutiao',
 }
 
 dry_run = False

From a666d0d9a5d923dc852525b337a9db9c7878c02d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 21:29:18 +0100
Subject: [PATCH 0561/1225] update README

---
 README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 403a288f6d..63d684a19f 100644
--- a/README.md
+++ b/README.md
@@ -359,6 +359,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Facebook    | <https://www.facebook.com/>   |✓| | |
 | eHow        | <http://www.ehow.com/>        |✓| | |
 | Dailymotion | <http://www.dailymotion.com/> |✓| | |
+| Coub        | <http://coub.com>             |✓| | |
 | CBS         | <http://www.cbs.com/>         |✓| | |
 | Bandcamp    | <http://bandcamp.com/>        | | |✓|
 | AliveThai   | <http://alive.in.th/>         |✓| | |
@@ -403,8 +404,9 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
-| 火猫TV   | <http://www.huomao.com/>         |✓| | |
-| 全民Tv   | <http://www.quanmin.tv/>         |✓| | |
+| 火猫TV   | <http://www.huomao.com/>       |✓| | |
+| 全民Tv   | <http://www.quanmin.tv/>       |✓| | |
+| 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 

From 5696308210d008dea20d8130710bcca1f04724a9 Mon Sep 17 00:00:00 2001
From: Sergey Zmushko <zmuxsm@gmail.com>
Date: Mon, 27 Nov 2017 23:27:52 +0300
Subject: [PATCH 0562/1225] [coub] added quality support

---
 src/you_get/extractors/coub.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/coub.py b/src/you_get/extractors/coub.py
index 7f5dccb665..44e403d320 100644
--- a/src/you_get/extractors/coub.py
+++ b/src/you_get/extractors/coub.py
@@ -58,9 +58,23 @@ def fix_coub_video_file(file_path):
 
 
 def get_title_and_urls(json_data):
-    title = legitimize(json_data['title'].replace(" ", "_"))
-    video_url = json_data['file_versions']['html5']['video']['high']['url']
-    audio_url = json_data['file_versions']['html5']['audio']['high']['url']
+    title = legitimize(re.sub('[\s*]', "_", json_data['title']))
+    video_info = json_data['file_versions']['html5']['video']
+    if 'high' not in video_info:
+        if 'med' not in video_info:
+            video_url = video_info['low']['url']
+        else:
+            video_url = video_info['med']['url']
+    else:
+        video_url = video_info['high']['url']
+    audio_info = json_data['file_versions']['html5']['audio']
+    if 'high' not in audio_info:
+        if 'med' not in audio_info:
+            audio_url = audio_info['low']['url']
+        else:
+            audio_url = audio_info['med']['url']
+    else:
+        audio_url = audio_info['high']['url']
     return title, video_url, audio_url
 
 

From 07d37bd9902fafc7bbb0e77f9673935d09e27249 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 21:36:22 +0100
Subject: [PATCH 0563/1225] update README

---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 63d684a19f..c432e2661c 100644
--- a/README.md
+++ b/README.md
@@ -109,7 +109,7 @@ $ brew install you-get
 
 ### Shell completion
 
-Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](contrib/completion). Please consult your shell's manual for how to take advantage of them.
+Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](https://github.com/soimort/you-get/tree/develop/contrib/completion). Please consult your shell's manual for how to take advantage of them.
 
 ## Upgrading
 
@@ -359,7 +359,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Facebook    | <https://www.facebook.com/>   |✓| | |
 | eHow        | <http://www.ehow.com/>        |✓| | |
 | Dailymotion | <http://www.dailymotion.com/> |✓| | |
-| Coub        | <http://coub.com>             |✓| | |
+| Coub        | <http://coub.com/>            |✓| | |
 | CBS         | <http://www.cbs.com/>         |✓| | |
 | Bandcamp    | <http://bandcamp.com/>        | | |✓|
 | AliveThai   | <http://alive.in.th/>         |✓| | |
@@ -405,7 +405,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
 | 火猫TV   | <http://www.huomao.com/>       |✓| | |
-| 全民Tv   | <http://www.quanmin.tv/>       |✓| | |
+| 全民直播 | <http://www.quanmin.tv/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.

From 414e2c3f0b583881a00232d5407838d120392f4c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 21:40:16 +0100
Subject: [PATCH 0564/1225] version 0.4.985

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 88315252c3..21f6d7ad8e 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.964'
+__version__ = '0.4.985'

From 75b44b83826b3c2d9a298df1dae3e2805b151205 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 23:43:12 +0100
Subject: [PATCH 0565/1225] add CONTRIBUTING.md

---
 CONTRIBUTING.md | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 CONTRIBUTING.md

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000000..b7b6ba42b6
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,27 @@
+# How to Contribute
+
+`you-get` is currently experimenting with an aggressive approach to handling issues. Namely, a bug report must be addressed with some code via a pull request.
+
+## Report a broken extractor
+
+**How-To:** Please open a new pull request with the following changes:
+
+* Add a new test case in [tests/test.py](https://github.com/soimort/you-get/blob/develop/tests/test.py), with the failing URL(s).
+
+The Travis CI build will (ideally) fail showing a :x:, which means you have successfully reported a broken extractor.
+
+Such a valid PR will be either *closed* if it's fixed by another PR, or *merged* if it's fixed by follow-up commits from the reporter himself/herself.
+
+## Report other issues / Suggest a new feature
+
+**How-To:** Please open a pull request with the proposed changes directly.
+
+A valid PR need not be complete (i.e., can be WIP), but it should contain at least one sensible, nontrivial commit.
+
+## Hints
+
+* The [`develop`](https://github.com/soimort/you-get/tree/develop) branch is where your pull request goes.
+* Remember to rebase.
+* Document your PR clearly, and if applicable, provide some sample links for reviewers to test with.
+* Write well-formatted, easy-to-understand commit messages. If you don't know how, look at existing ones.
+* We will not ask you to sign a CLA, but you must assure that your code can be legally redistributed (under the terms of the MIT license).

From 9837a1f949e5532dcf231840d7e2bbd51e160059 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Nov 2017 23:54:37 +0100
Subject: [PATCH 0566/1225] update README

---
 README.md | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index c432e2661c..6ec75a57d6 100644
--- a/README.md
+++ b/README.md
@@ -414,19 +414,13 @@ For all other sites not on the list, the universal extractor will take care of f
 
 If something is broken and `you-get` can't get you things you want, don't panic. (Yes, this happens all the time!)
 
-Check if it's already a known problem on <https://github.com/soimort/you-get/wiki/Known-Bugs>, and search on the [list of open issues](https://github.com/soimort/you-get/issues). If it has not been reported yet, open a new issue, with detailed command-line output attached.
+Check if it's already a known problem on <https://github.com/soimort/you-get/wiki/Known-Bugs>. If not, follow the guidelines on [how to report a broken extractor](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md#report-a-broken-extractor).
 
 ## Getting Involved
 
 You can reach us on the Gitter channel [#soimort/you-get](https://gitter.im/soimort/you-get) (here's how you [set up your IRC client](http://irc.gitter.im) for Gitter). If you have a quick question regarding `you-get`, ask it there.
 
-All kinds of pull requests are welcome. However, there are a few guidelines to follow:
-
-* The [`develop`](https://github.com/soimort/you-get/tree/develop) branch is where your pull request should go.
-* Remember to rebase.
-* Document your PR clearly, and if applicable, provide some sample links for reviewers to test with.
-* Write well-formatted, easy-to-understand commit messages. If you don't know how, look at existing ones.
-* We will not ask you to sign a CLA, but you must assure that your code can be legally redistributed (under the terms of the MIT license).
+If you are seeking to report an issue or contribute, please make sure to read [the guidelines](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) first.
 
 ## Legal Issues
 

From 64772ce2e92f37318efb39ace2e55c18b76c804c Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Wed, 29 Nov 2017 17:11:09 +0800
Subject: [PATCH 0567/1225] Update common.py

File "xxx/site-packages/you_get/common.py", line 750, in update
    sys.stdout.write('\r' + bar)
UnicodeEncodeError: 'ascii' codec can't encode characters in position 23-64: ordinal not in range(128)
---
 src/you_get/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5c69f03016..bc176f6753 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+import io
 import os
 import re
 import sys
@@ -18,7 +19,7 @@
 from .util.git import get_version
 from .util.strings import get_filename, unescape_html
 from . import json_output as json_output_
-
+sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf8')
 
 SITES = {
     '163'              : 'netease',

From 795e7a037f85ba9d5e4957b16d61e9d57db4c245 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 30 Nov 2017 12:31:27 +0100
Subject: [PATCH 0568/1225] [youku] update ccode, fix #2528

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index cd4483a1ad..13a18a2e0a 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0406'
+        self.ccode = '0501'
         self.utid = None
 
     def youku_ups(self):
@@ -154,7 +154,7 @@ def prepare(self, **kwargs):
                     log.wtf('Cannot fetch vid')
 
         if kwargs.get('src') and kwargs['src'] == 'tudou':
-            self.ccode = '0406'
+            self.ccode = '0501'
 
         if kwargs.get('password') and kwargs['password']:
             self.password_protected = True

From a4910db9b42a47b2d790c6bd9bf609baec62d11a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 2 Dec 2017 01:35:43 +0100
Subject: [PATCH 0569/1225] version 0.4.990

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 21f6d7ad8e..29d1943eda 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.985'
+__version__ = '0.4.990'

From 897e941bbd0eb0ffa420ce885d442ee762337ce0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 3 Dec 2017 15:34:49 +0100
Subject: [PATCH 0570/1225] [youku] update ccode

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 13a18a2e0a..2e87b24226 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0501'
+        self.ccode = '0507'
         self.utid = None
 
     def youku_ups(self):

From f668b7062ad8abe35de15a317ae2f755a3f4ae44 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Dec 2017 03:04:32 +0100
Subject: [PATCH 0571/1225] [bilibili] PEP 8 formatting

---
 src/you_get/extractors/bilibili.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 3b5aabea3b..305b3b6990 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -87,7 +87,7 @@ def parse_bili_xml(self, xml_str):
 
     def download_by_vid(self, cid, bangumi, **kwargs):
         stream_id = kwargs.get('stream_id')
-# guard here. if stream_id invalid, fallback as not stream_id
+        # guard here. if stream_id invalid, fallback as not stream_id
         if stream_id and stream_id in self.fmt2qlt:
             quality = stream_id
         else:
@@ -95,7 +95,7 @@ def download_by_vid(self, cid, bangumi, **kwargs):
 
         info_only = kwargs.get('info_only')
         if not info_only or stream_id:
-# won't be None
+            # won't be None
             qlt = self.fmt2qlt.get(quality)
             api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
             self.parse_bili_xml(api_xml)
@@ -117,7 +117,7 @@ def prepare(self, **kwargs):
         self.ua = fake_headers['User-Agent']
         self.url = url_locations([self.url])[0]
         frag = urllib.parse.urlparse(self.url).fragment
-# http://www.bilibili.com/video/av3141144/index_2.html#page=3
+        # http://www.bilibili.com/video/av3141144/index_2.html#page=3
         if frag:
             hit = re.search(r'page=(\d+)', frag)
             if hit is not None:
@@ -153,12 +153,12 @@ def movie_entry(self, **kwargs):
         patt = r"var\s*aid\s*=\s*'(\d+)'"
         aid = re.search(patt, self.page).group(1)
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
-# better ideas for bangumi_movie titles?
+        # better ideas for bangumi_movie titles?
         self.title = page_list[0]['pagename']
         self.download_by_vid(page_list[0]['cid'], True, bangumi_movie=True, **kwargs)
 
     def entry(self, **kwargs):
-# tencent player
+        # tencent player
         tc_flashvars = re.search(r'"bili-cid=\d+&bili-aid=\d+&vid=([^"]+)"', self.page)
         if tc_flashvars:
             tc_flashvars = tc_flashvars.group(1)
@@ -171,7 +171,7 @@ def entry(self, **kwargs):
         if cid is not None:
             self.download_by_vid(cid, False, **kwargs)
         else:
-# flashvars?
+            # flashvars?
             flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)
             if flashvars is None:
                 raise Exception('Unsupported page {}'.format(self.url))
@@ -327,7 +327,7 @@ def parse_cid_playurl(xml):
 
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url])[0]
-# a bangumi here? possible?
+    # a bangumi here? possible?
     if 'live.bilibili' in url:
         site.download_by_url(url)
     elif 'bangumi.bilibili' in url:

From 731fcb5eac39ce2064e00e08f09633567b52a072 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Dec 2017 04:01:08 +0100
Subject: [PATCH 0572/1225] [bilibili] wubba lubba dub dub

---
 src/you_get/extractors/bilibili.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 305b3b6990..b423c8beeb 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -94,16 +94,11 @@ def download_by_vid(self, cid, bangumi, **kwargs):
             quality = 'hdflv' if bangumi else 'flv'
 
         info_only = kwargs.get('info_only')
-        if not info_only or stream_id:
-            # won't be None
-            qlt = self.fmt2qlt.get(quality)
+        for qlt in range(4, -1, -1):
             api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
             self.parse_bili_xml(api_xml)
+        if not info_only or stream_id:
             self.danmuku = get_danmuku_xml(cid)
-        else:
-            for qlt in range(4, 0, -1):
-                api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
-                self.parse_bili_xml(api_xml)
 
     def prepare(self, **kwargs):
         if socket.getdefaulttimeout() == 600: # no timeout specified

From f6b212397ff387088ba2a10e9d22b88361cfba42 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Dec 2017 04:18:06 +0100
Subject: [PATCH 0573/1225] elephants are cool

---
 README.md | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 6ec75a57d6..4b9045c3d0 100644
--- a/README.md
+++ b/README.md
@@ -6,17 +6,23 @@
 
 [You-Get](https://you-get.org/) is a tiny command-line utility to download media contents (videos, audios, images) from the Web, in case there is no other handy way to do it.
 
-Here's how you use `you-get` to download a video from [this web page](http://www.fsf.org/blogs/rms/20140407-geneva-tedx-talk-free-software-free-society):
+Here's how you use `you-get` to download a video from [YouTube](https://www.youtube.com/watch?v=jNQXAC9IVRw):
 
 ```console
-$ you-get http://www.fsf.org/blogs/rms/20140407-geneva-tedx-talk-free-software-free-society
-Site:       fsf.org
-Title:      TEDxGE2014_Stallman05_LQ
-Type:       WebM video (video/webm)
-Size:       27.12 MiB (28435804 Bytes)
-
-Downloading TEDxGE2014_Stallman05_LQ.webm ...
-100.0% ( 27.1/27.1 MB) ├████████████████████████████████████████┤[1/1]   12 MB/s
+$ you-get 'https://www.youtube.com/watch?v=jNQXAC9IVRw'
+site:                YouTube
+title:               Me at the zoo
+stream:
+    - itag:          43
+      container:     webm
+      quality:       medium
+      size:          0.5 MiB (564215 bytes)
+    # download-with: you-get --itag=43 [URL]
+
+Downloading Me at the zoo.webm ...
+ 100% (  0.5/  0.5MB) ├██████████████████████████████████┤[1/1]    6 MB/s
+
+Saving Me at the zoo.en.srt ... Done.
 ```
 
 And here's why you might want to use it:

From 501dbe1856a016944adc86d5c5a38f71a2d87ae0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Dec 2017 04:18:45 +0100
Subject: [PATCH 0574/1225] version 0.4.995

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 29d1943eda..6301dbc881 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.990'
+__version__ = '0.4.995'

From 3d15a3afdf038675014cedb028f0405198b0ddfc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Dec 2017 14:17:11 +0100
Subject: [PATCH 0575/1225] [bilibili] flv720

---
 src/you_get/extractors/bilibili.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index b423c8beeb..8cee2d48f2 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -32,6 +32,7 @@ class Bilibili(VideoExtractor):
     SEC2 = '9b288147e5474dd2aa67085f716c560d'
     stream_types = [
             {'id': 'hdflv'},
+            {'id': 'flv720'},
             {'id': 'flv'},
             {'id': 'hdmp4'},
             {'id': 'mp4'},
@@ -43,13 +44,15 @@ class Bilibili(VideoExtractor):
     @staticmethod
     def bilibili_stream_type(urls):
         url = urls[0]
-        if 'hd.flv?' in url or '-112.flv' in url:
+        if 'hd.flv' in url or '-112.flv' in url:
             return 'hdflv', 'flv'
-        if '.flv?' in url:
+        if '-64.flv' in url:
+            return 'flv720', 'flv'
+        if '.flv' in url:
             return 'flv', 'flv'
-        if 'hd.mp4?' in url or '-48.mp4' in url:
+        if 'hd.mp4' in url or '-48.mp4' in url:
             return 'hdmp4', 'mp4'
-        if '.mp4?' in url:
+        if '.mp4' in url:
             return 'mp4', 'mp4'
         raise Exception('Unknown stream type')
 

From 219ac3ce62cbebf7a12882086d080d184248d625 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 5 Dec 2017 15:13:56 +0100
Subject: [PATCH 0576/1225] [youku] update ccode

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 2e87b24226..0e0d724001 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0507'
+        self.ccode = '0508'
         self.utid = None
 
     def youku_ups(self):

From fa39bc3012f21f5bfb390854988d05d787ac2188 Mon Sep 17 00:00:00 2001
From: iawia002 <z2d@jifangcheng.com>
Date: Thu, 7 Dec 2017 16:35:56 +0800
Subject: [PATCH 0577/1225] update the test

---
 tests/test.py | 43 ++++++++++++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 9 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 84fb9cae66..a4503cb90b 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -2,25 +2,50 @@
 
 import unittest
 
-from you_get.extractors import *
+from you_get.extractors import (
+    imgur,
+    magisto,
+    youtube,
+    yixia,
+    bilibili,
+)
 
 
 class YouGetTests(unittest.TestCase):
     def test_imgur(self):
-        imgur.download("http://imgur.com/WVLk5nD", info_only=True)
-        imgur.download("http://imgur.com/gallery/WVLk5nD", info_only=True)
+        imgur.download('http://imgur.com/WVLk5nD', info_only=True)
+        imgur.download('http://imgur.com/gallery/WVLk5nD', info_only=True)
 
     def test_magisto(self):
-        magisto.download("http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA", info_only=True)
+        magisto.download(
+            'http://www.magisto.com/album/video/f3x9AAQORAkfDnIFDA',
+            info_only=True
+        )
 
     def test_youtube(self):
-        youtube.download("http://www.youtube.com/watch?v=pzKerr0JIPA", info_only=True)
-        youtube.download("http://youtu.be/pzKerr0JIPA", info_only=True)
-        youtube.download("http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare",
-                         info_only=True)
+        youtube.download(
+            'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True
+        )
+        youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True)
+        youtube.download(
+            'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
+            info_only=True
+        )
 
     def test_yixia(self):
-        yixia_download("http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8", info_only=True)
+        yixia.download(
+            'http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8',  # noqa
+            info_only=True
+        )
+
+    def test_bilibili(self):
+        bilibili.download(
+            'https://www.bilibili.com/video/av16907446/', info_only=True
+        )
+        bilibili.download(
+            'https://www.bilibili.com/video/av13228063/', info_only=True
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

From 88fbc212aeeec2b9c6085c7ce1bf33675f68366c Mon Sep 17 00:00:00 2001
From: iawia002 <z2d@jifangcheng.com>
Date: Sat, 9 Dec 2017 18:05:13 +0800
Subject: [PATCH 0578/1225] fix bilibili title regex

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 8cee2d48f2..59b9b07e47 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -125,7 +125,7 @@ def prepare(self, **kwargs):
         self.referer = self.url
         self.page = get_content(self.url)
 
-        m = re.search(r'<h1\s*title="([^"]+)"', self.page)
+        m = re.search(r'<h1.*?>(.*?)</h1>', self.page)
         if m is not None:
             self.title = m.group(1)
         if self.title is None:

From 6c3bdfd90ce3da05582ed06a6a914651e88341eb Mon Sep 17 00:00:00 2001
From: perror <15058342792@163.com>
Date: Sun, 10 Dec 2017 13:14:32 +0800
Subject: [PATCH 0579/1225] New site support: https://www.ixigua.com/

---
 README.md                        |  1 +
 src/you_get/common.py            |  1 +
 src/you_get/extractors/ixigua.py | 85 ++++++++++++++++++++++++++++++++
 3 files changed, 87 insertions(+)
 create mode 100644 src/you_get/extractors/ixigua.py

diff --git a/README.md b/README.md
index 4b9045c3d0..2158c80fc3 100644
--- a/README.md
+++ b/README.md
@@ -413,6 +413,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 火猫TV   | <http://www.huomao.com/>       |✓| | |
 | 全民直播 | <http://www.quanmin.tv/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
+| 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index bc176f6753..8abfafd543 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -61,6 +61,7 @@
     'interest'         : 'interest',
     'iqilu'            : 'iqilu',
     'iqiyi'            : 'iqiyi',
+    'ixigua'           : 'ixigua',
     'isuntv'           : 'suntv',
     'joy'              : 'joy',
     'kankanews'        : 'bilibili',
diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
new file mode 100644
index 0000000000..aaed195dbe
--- /dev/null
+++ b/src/you_get/extractors/ixigua.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+__all__ = ['ixigua_download', 'ixigua_download_playlist']
+import base64
+import random
+import binascii
+from ..common import *
+
+def get_video_id(text):
+    re_id = r"videoId: '(.*?)'"
+    return re.findall(re_id, text)[0]
+
+def get_r():
+    return str(random.random())[2:]
+
+def right_shift(val, n):
+    return val >> n if val >= 0 else (val + 0x100000000) >> n
+
+def get_s(text):
+    """get video info"""
+    id = get_video_id(text)
+    p = get_r()
+    url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % id
+    n = parse.urlparse(url).path + '?r=%s' % p
+    c = binascii.crc32(n.encode('utf-8'))
+    s = right_shift(c, 0)
+    title = ''.join(re.findall(r"title: '(.*?)',", text))
+    return url + '?r=%s&s=%s' % (p, s), title
+
+def get_moment(url, user_id, base_url, video_list):
+    """Recursively obtaining a video list"""
+    video_list_data = json.loads(get_content(url))
+    if not video_list_data['next']['max_behot_time']:
+        return video_list
+    [video_list.append(i["display_url"]) for i in video_list_data["data"]]
+    max_behot_time = video_list_data['next']['max_behot_time']
+    _param = {
+        'user_id': user_id,
+        'base_url': base_url,
+        'video_list': video_list,
+        'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
+    }
+    return get_moment(**_param)
+
+def ixigua_download(url, output_dir='.', info_only=False, **kwargs):
+    """ Download a single video
+        Sample URL: https://www.ixigua.com/a6487187567887254029/#mid=59051127876
+    """
+    try:
+        video_info_url, title = get_s(get_content(url))
+        video_info = json.loads(get_content(video_info_url))
+    except Exception:
+        raise NotImplementedError(url)
+    try:
+        video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"]).decode()
+    except Exception:
+        raise NotImplementedError(url)
+    filetype, ext, size = url_info(video_url)
+    print_info(site_info, title, filetype, size)
+    if not info_only:
+        download_urls([video_url], title, ext, size, output_dir=output_dir)
+
+def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs):
+    """Download all video from the user's video list
+        Sample URL: https://www.ixigua.com/c/user/71141690831/
+    """
+    if 'user' not in url:
+        raise NotImplementedError(url)
+    user_id = url.split('/')[-2]
+    max_behot_time = 0
+    if not user_id:
+        raise NotImplementedError(url)
+    base_url = "https://www.ixigua.com/c/user/article/?user_id={user_id}" \
+               "&max_behot_time={max_behot_time}&max_repin_time=0&count=20&page_type=0"
+    _param = {
+        'user_id': user_id,
+        'base_url': base_url,
+        'video_list': [],
+        'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
+    }
+    for i in get_moment(**_param):
+        ixigua_download(i, output_dir, info_only, **kwargs)
+
+site_info = "ixigua.com"
+download = ixigua_download
+download_playlist = ixigua_download_playlist
\ No newline at end of file

From 7b996ec75517b1e4ca3498e34ef3d06b68f9aef4 Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Tue, 12 Dec 2017 12:43:20 +0800
Subject: [PATCH 0580/1225] Fix download_url_ffmpeg extension

---
 src/you_get/common.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8abfafd543..2f0e876f2b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1022,8 +1022,11 @@ def download_url_ffmpeg(
     global output_filename
     if output_filename:
         dotPos = output_filename.rfind('.')
-        title = output_filename[:dotPos]
-        ext = output_filename[dotPos+1:]
+        if dotPos > 0:
+            title = output_filename[:dotPos]
+            ext = output_filename[dotPos+1:]
+        else:
+            title = output_filename
 
     title = tr(get_filename(title))
 

From 3032ec96796e2c6c734a3434373684037de7321a Mon Sep 17 00:00:00 2001
From: moorefu <chengmu.fu@gmail.com>
Date: Wed, 13 Dec 2017 11:52:05 +0800
Subject: [PATCH 0581/1225] New site support: https://www.kuaishou.com/

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/kuaishou.py | 39 ++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 src/you_get/extractors/kuaishou.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2f0e876f2b..ce707ef46a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -67,6 +67,7 @@
     'kankanews'        : 'bilibili',
     'khanacademy'      : 'khan',
     'ku6'              : 'ku6',
+    'kuaishou'         : 'kuaishou',
     'kugou'            : 'kugou',
     'kuwo'             : 'kuwo',
     'le'               : 'le',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index f1acf00039..333cc03c8e 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -85,3 +85,4 @@
 from .ted import *
 from .khan import *
 from .zhanqi import *
+from .kuaishou import *
diff --git a/src/you_get/extractors/kuaishou.py b/src/you_get/extractors/kuaishou.py
new file mode 100644
index 0000000000..a21f8ffa64
--- /dev/null
+++ b/src/you_get/extractors/kuaishou.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import urllib.request
+import urllib.parse
+import json
+import re
+
+from ..util import log
+from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size
+
+__all__ = ['kuaishou_download_by_url']
+
+
+def kuaishou_download_by_url(url, info_only=False, **kwargs):
+    page = get_content(url)
+    # size = video_list[-1]['size']
+    # result wrong size
+    try:
+        og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content=\"(.+?)\"/>", page).group(1)
+        video_url = og_video_url
+        title = url.split('/')[-1]
+        size = url_size(video_url)
+        video_format = video_url.split('.')[-1]
+        print_info(site_info, title, video_format, size)
+        if not info_only:
+            download_urls([video_url], title, video_format, size, **kwargs)
+    except:# extract image
+        og_image_url = re.search(r"<meta\s+property=\"og:image\"\s+content=\"(.+?)\"/>", page).group(1)
+        image_url = og_image_url
+        title = url.split('/')[-1]
+        size = url_size(image_url)
+        image_format = image_url.split('.')[-1]
+        print_info(site_info, title, image_format, size)
+        if not info_only:
+            download_urls([image_url], title, image_format, size, **kwargs)
+
+site_info = "kuaishou.com"
+download = kuaishou_download_by_url
+download_playlist = playlist_not_supported('kuaishou')

From 4ce28ac2b57e5d7383c970c5e42819c672e4cac8 Mon Sep 17 00:00:00 2001
From: iawia002 <z2d@jifangcheng.com>
Date: Fri, 15 Dec 2017 17:21:34 +0800
Subject: [PATCH 0582/1225] =?UTF-8?q?=E2=9C=A8=20add=20douyin=20support?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/douyin.py   | 38 ++++++++++++++++++++++++++++++
 tests/test.py                      |  7 ++++++
 5 files changed, 48 insertions(+)
 create mode 100644 src/you_get/extractors/douyin.py

diff --git a/README.md b/README.md
index 2158c80fc3..5e338212d1 100644
--- a/README.md
+++ b/README.md
@@ -414,6 +414,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 全民直播 | <http://www.quanmin.tv/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
+| 抖音 | <https://www.douyin.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index ce707ef46a..32d831e507 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -38,6 +38,7 @@
     'dailymotion'      : 'dailymotion',
     'dilidili'         : 'dilidili',
     'douban'           : 'douban',
+    'douyin'           : 'douyin',
     'douyu'            : 'douyutv',
     'ehow'             : 'ehow',
     'facebook'         : 'facebook',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 333cc03c8e..46e5c89c67 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -15,6 +15,7 @@
 from .dailymotion import *
 from .dilidili import *
 from .douban import *
+from .douyin import *
 from .douyutv import *
 from .ehow import *
 from .facebook import *
diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
new file mode 100644
index 0000000000..213837e772
--- /dev/null
+++ b/src/you_get/extractors/douyin.py
@@ -0,0 +1,38 @@
+# coding=utf-8
+
+import re
+import json
+
+from ..common import (
+    url_size,
+    print_info,
+    get_content,
+    download_urls,
+    playlist_not_supported,
+)
+
+
+__all__ = ['douyin_download_by_url']
+
+
+def douyin_download_by_url(url, **kwargs):
+    page_content = get_content(url)
+    match_rule = re.compile(r'var data = \[(.*?)\];')
+    video_info = json.loads(match_rule.findall(page_content)[0])
+    video_url = video_info['video']['play_addr']['url_list'][0]
+    title = video_info['cha_list'][0]['cha_name']
+    video_format = 'mp4'
+    size = url_size(video_url)
+    print_info(
+        site_info='douyin.com', title=title,
+        type=video_format, size=size
+    )
+    if not kwargs['info_only']:
+        download_urls(
+            urls=[video_url], title=title, ext=video_format, total_size=size,
+            **kwargs
+        )
+
+
+download = douyin_download_by_url
+download_playlist = playlist_not_supported('douyin')
diff --git a/tests/test.py b/tests/test.py
index a4503cb90b..36c699f5c6 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -8,6 +8,7 @@
     youtube,
     yixia,
     bilibili,
+    douyin,
 )
 
 
@@ -46,6 +47,12 @@ def test_bilibili(self):
             'https://www.bilibili.com/video/av13228063/', info_only=True
         )
 
+    def test_douyin(self):
+        douyin.download(
+            'https://www.douyin.com/share/video/6492273288897629454',
+            info_only=True
+        )
+
 
 if __name__ == '__main__':
     unittest.main()

From cd77aad47cfe2af1250f77d1e83c96650fafa081 Mon Sep 17 00:00:00 2001
From: iawia002 <z2d@jifangcheng.com>
Date: Fri, 15 Dec 2017 22:30:34 +0800
Subject: [PATCH 0583/1225] =?UTF-8?q?=F0=9F=94=A7=20add=20missing=20intr?=
 =?UTF-8?q?=20for=20kuaishou?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 5e338212d1..86c5e4e958 100644
--- a/README.md
+++ b/README.md
@@ -414,6 +414,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 全民直播 | <http://www.quanmin.tv/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
+| 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.

From 65f8cec8e2668749a0684afbd1a50c61c4c22761 Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Tue, 19 Dec 2017 18:53:16 +0800
Subject: [PATCH 0584/1225] fix json

---
 src/you_get/extractors/le.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index 2cd5f16190..1914810430 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -67,7 +67,7 @@ def video_info(vid,**kwargs):
     else:
         print("Current Video Supports:")
         for i in support_stream_id:
-            print("\t--format",i,"<URL>")
+            # print("\t--format",i,"<URL>")
         if "1080p" in support_stream_id:
             stream_id = '1080p'
         elif "720p" in support_stream_id:

From cb0dc09a01cb4f729f009992fb2a63f848502593 Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Tue, 19 Dec 2017 18:56:16 +0800
Subject: [PATCH 0585/1225] fix json

---
 src/you_get/extractors/le.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index 1914810430..940b9b350b 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -65,9 +65,6 @@ def video_info(vid,**kwargs):
     if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id:
         stream_id = kwargs["stream_id"]
     else:
-        print("Current Video Supports:")
-        for i in support_stream_id:
-            # print("\t--format",i,"<URL>")
         if "1080p" in support_stream_id:
             stream_id = '1080p'
         elif "720p" in support_stream_id:

From 1138f17bfb78e51fa8c82cb2242c74bd3f18687a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Dec 2017 18:05:28 +0100
Subject: [PATCH 0586/1225] [test] remove test_douyin

---
 tests/test.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 36c699f5c6..699675f3c6 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -47,12 +47,6 @@ def test_bilibili(self):
             'https://www.bilibili.com/video/av13228063/', info_only=True
         )
 
-    def test_douyin(self):
-        douyin.download(
-            'https://www.douyin.com/share/video/6492273288897629454',
-            info_only=True
-        )
-
 
 if __name__ == '__main__':
     unittest.main()

From 9d7c9f96510e53f2ca308da1a34e92d1dba52155 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Dec 2017 14:12:56 +0100
Subject: [PATCH 0587/1225] [bilibili] new bangumi URL (fix #2543)

---
 src/you_get/extractors/bilibili.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 59b9b07e47..57661766ae 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -165,7 +165,10 @@ def entry(self, **kwargs):
             qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
             return
 
-        cid = re.search(r'cid=(\d+)', self.page).group(1)
+        try:
+            cid = re.search(r'cid=(\d+)', self.page).group(1)
+        except:
+            cid = re.search(r'"cid":(\d+)', self.page).group(1)
         if cid is not None:
             self.download_by_vid(cid, False, **kwargs)
         else:

From 6baa80419c22bbbb906cb7d57e7712eb63018a12 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Dec 2017 17:16:43 +0100
Subject: [PATCH 0588/1225] [youku] update ccode

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 0e0d724001..976ed44774 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0508'
+        self.ccode = '0512'
         self.utid = None
 
     def youku_ups(self):
@@ -154,7 +154,7 @@ def prepare(self, **kwargs):
                     log.wtf('Cannot fetch vid')
 
         if kwargs.get('src') and kwargs['src'] == 'tudou':
-            self.ccode = '0501'
+            self.ccode = '0512'
 
         if kwargs.get('password') and kwargs['password']:
             self.password_protected = True

From c0c76915dc3e0c21972628927105abdb71f286cb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Dec 2017 18:27:08 +0100
Subject: [PATCH 0589/1225] version 0.4.1011

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 6301dbc881..e4420b84d7 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.995'
+__version__ = '0.4.1011'

From e7954c4149170d3ff42cdab69ac03931c5afa6c0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Dec 2017 22:02:11 +0100
Subject: [PATCH 0590/1225] [bilibili] warn when target URL is a playlist

---
 src/you_get/extractors/bilibili.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 57661766ae..76aad869dc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -165,6 +165,10 @@ def entry(self, **kwargs):
             qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
             return
 
+        has_plist = re.search(r'<option', self.page)
+        if has_plist and r1('index_(\d+).html', self.url) is None:
+            log.w('This page contains a playlist. (use --playlist to download all videos.)')
+
         try:
             cid = re.search(r'cid=(\d+)', self.page).group(1)
         except:

From 29d5201a9bdb25c7b6481c1be043d5c29cfd091a Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Mon, 25 Dec 2017 18:39:10 +0800
Subject: [PATCH 0591/1225] fix
 https://v.qq.com/x/cover/9hpjiv5fhiyn86u/t0522x58xma.html

---
 src/you_get/extractors/qq.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 537dffd27c..438c1f2401 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -36,13 +36,18 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
         if key_json.get('key') is None:
+            vkey = video_json['vl']['vi'][0]['fvkey']
+            url = '{}{}?vkey={}'.format(video_json['vl']['vi'][0]['ul']['ui'][0]['url'], fn_pre + '.mp4', vkey)
+        else:
+            vkey = key_json['key']
+            url = '{}{}?vkey={}'.format(host, filename, vkey)
+        if not vkey:
             if part == 1:
                 log.wtf(key_json['msg'])
             else:
                 log.w(key_json['msg'])
             break
-        vkey = key_json['key']
-        url = '{}{}?vkey={}'.format(host, filename, vkey)
+
         part_urls.append(url)
         _, ext, size = url_info(url)
         total_size += size
@@ -140,9 +145,13 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     else:
         content = get_content(url)
         #vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
-        vid = url.split('/')[-1].split('.')[0] #https://v.qq.com/x/cover/ps6mnfqyrfo7es3/q0181hpdvo5.html?
+        rurl = match1(content, r'<link.*?rel\s*=\s*"canonical".*?href\s*="(.+?)".*?>') #https://v.qq.com/x/cover/9hpjiv5fhiyn86u/t0522x58xma.html
+        vid = ""
+        if rurl:
+            vid = rurl.split('/')[-1].split('.')[0]
+        vid = vid if vid else url.split('/')[-1].split('.')[0] #https://v.qq.com/x/cover/ps6mnfqyrfo7es3/q0181hpdvo5.html?
         vid = vid if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
-        if vid is None:
+        if not vid:
             vid = match1(content, r'id"*\s*:\s*"(.+?)"')
         title = match1(content,r'<a.*?id\s*=\s*"%s".*?title\s*=\s*"(.+?)".*?>'%vid)
         title = match1(content, r'title">([^"]+)</p>') if not title else title

From 6e17b0c7f9499ba8b7467d5183a34835b2b10944 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 30 Dec 2017 20:44:31 +0100
Subject: [PATCH 0592/1225] [bilibili] enable bangumi API

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 76aad869dc..09e08ecb74 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -174,7 +174,7 @@ def entry(self, **kwargs):
         except:
             cid = re.search(r'"cid":(\d+)', self.page).group(1)
         if cid is not None:
-            self.download_by_vid(cid, False, **kwargs)
+            self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs)
         else:
             # flashvars?
             flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)

From a1290be2ec14943c6a77cbb4d80c561083cee44e Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Wed, 3 Jan 2018 11:43:26 +0800
Subject: [PATCH 0593/1225] fix https://v.qq.com/x/page/q0527wsyqpv.html

---
 src/you_get/extractors/qq.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 438c1f2401..199df921ac 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -149,6 +149,8 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = ""
         if rurl:
             vid = rurl.split('/')[-1].split('.')[0]
+            if vid == "undefined":
+                vid = ""
         vid = vid if vid else url.split('/')[-1].split('.')[0] #https://v.qq.com/x/cover/ps6mnfqyrfo7es3/q0181hpdvo5.html?
         vid = vid if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback
         if not vid:

From 3d08064d315949bff6ec014a059ad889a248adf1 Mon Sep 17 00:00:00 2001
From: Hongbo Liu <hbliu@freewheel.tv>
Date: Thu, 4 Jan 2018 23:34:20 +0800
Subject: [PATCH 0594/1225] [qq] fix weixin embedded video support

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 438c1f2401..89dd7b61c6 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -114,7 +114,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             qieDownload(url, output_dir=output_dir, merge=merge, info_only=info_only)
         return
 
-    if 'mp.weixin.qq.com/s?' in url:
+    if 'mp.weixin.qq.com/s' in url:
         content = get_content(url)
         vids = matchall(content, [r'\?vid=(\w+)'])
         for vid in vids:

From 68a0e9c4b0120cfa309d1959d74f4cc0685d5ab0 Mon Sep 17 00:00:00 2001
From: ygmpkk <ygmpkk@gmail.com>
Date: Fri, 5 Jan 2018 12:23:02 +0800
Subject: [PATCH 0595/1225] fix http code 403 and 466 error for bilibili

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 09e08ecb74..1ec41203ec 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -68,7 +68,7 @@ def api_req(self, cid, quality, bangumi, bangumi_movie=False, **kwargs):
             chksum = hashlib.md5(bytes(params_str+self.SEC2, 'utf8')).hexdigest()
             api_url = self.bangumi_api_url + params_str + '&sign=' + chksum
 
-        xml_str = get_content(api_url)
+        xml_str = get_content(api_url, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
         return xml_str
 
     def parse_bili_xml(self, xml_str):

From a345635473c9b2c2a1cb122f16bad4fae734f4c7 Mon Sep 17 00:00:00 2001
From: 9yw <25059561+9yw@users.noreply.github.com>
Date: Wed, 10 Jan 2018 15:32:40 +0800
Subject: [PATCH 0596/1225] Update youku.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ccode=0512会报错 客户端无权播放,201 改成5013可以下载
---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 976ed44774..9d74b9c88c 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0512'
+        self.ccode = '0513'
         self.utid = None
 
     def youku_ups(self):

From e8385913d91ba7b996bae19eb7623be0c3c7b36a Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Thu, 25 Jan 2018 23:37:46 +0800
Subject: [PATCH 0597/1225] update douyutv.py for room_id bugs

m.douyu.com DOES have the unique room_id, for counter-example: https://www.douyu.com/aleng1106, there is no room_id in this page
---
 src/you_get/extractors/douyutv.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 3b64066afc..4bd0e14cf9 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -43,8 +43,10 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     headers = {
         'user-agent': 'Mozilla/5.0 (iPad; CPU OS 8_1_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B466 Safari/600.1.4'
     }
+    
+    url = re.sub(r'[w.]*douyu.com','m.douyu.com',url)
     html = get_content(url, headers)
-    room_id_patt = r'"room_id"\s*:\s*(\d+),'
+    room_id_patt = r'room_id\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
     if room_id == "0":
         room_id = url[url.rfind('/')+1:]

From 34b51cbecb7e5044580ccb3458ceaf0b99c5959a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 2 Feb 2018 19:01:39 +0100
Subject: [PATCH 0598/1225] [bilibili] fix some title regexes

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1ec41203ec..ab19c41cad 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -125,11 +125,11 @@ def prepare(self, **kwargs):
         self.referer = self.url
         self.page = get_content(self.url)
 
-        m = re.search(r'<h1.*?>(.*?)</h1>', self.page)
+        m = re.search(r'<h1.*?>(.*?)</h1>', self.page) or re.search(r'<h1 title="([^"]+)">', self.page)
         if m is not None:
             self.title = m.group(1)
         if self.title is None:
-            m = re.search(r'<meta property="og:title" content="([^"]+)">', self.page)
+            m = re.search(r'property="og:title" content="([^"]+)"', self.page)
             if m is not None:
                 self.title = m.group(1)
         if 'subtitle' in kwargs:

From 38737c40f1541987b0c508803b75e1a12f8a5196 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Feb 2018 17:57:04 +0100
Subject: [PATCH 0599/1225] [test] remove test_yixia

---
 tests/test.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 699675f3c6..6562d7cad4 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -6,9 +6,7 @@
     imgur,
     magisto,
     youtube,
-    yixia,
     bilibili,
-    douyin,
 )
 
 
@@ -33,12 +31,6 @@ def test_youtube(self):
             info_only=True
         )
 
-    def test_yixia(self):
-        yixia.download(
-            'http://m.miaopai.com/show/channel/vlvreCo4OZiNdk5Jn1WvdopmAvdIJwi8',  # noqa
-            info_only=True
-        )
-
     def test_bilibili(self):
         bilibili.download(
             'https://www.bilibili.com/video/av16907446/', info_only=True

From ea0bf8c32a212205752f067d1dd06593a280b9f7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 9 Feb 2018 15:50:09 +0100
Subject: [PATCH 0600/1225] version 0.4.1025

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e4420b84d7..2d4ff9d03a 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1011'
+__version__ = '0.4.1025'

From 6bac1cd5dc2540ed140b03f3ea96e0cbe7add2f1 Mon Sep 17 00:00:00 2001
From: liushuyu <liushuyu011@gmail.com>
Date: Fri, 9 Feb 2018 15:53:04 -0700
Subject: [PATCH 0601/1225] bilibili: adapt to new bangumi API

---
 src/you_get/extractors/bilibili.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index ab19c41cad..e5abccab6b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -140,6 +140,8 @@ def prepare(self, **kwargs):
             self.movie_entry(**kwargs)
         elif 'bangumi.bilibili.com' in self.url:
             self.bangumi_entry(**kwargs)
+        elif 'bangumi/' in self.url:
+            self.bangumi_entry(**kwargs)
         elif 'live.bilibili.com' in self.url:
             self.live_entry(**kwargs)
         elif 'vc.bilibili.com' in self.url:
@@ -235,22 +237,22 @@ def vc_entry(self, **kwargs):
 
     def bangumi_entry(self, **kwargs):
         bangumi_id = re.search(r'(\d+)', self.url).group(1)
-        bangumi_data = get_bangumi_info(bangumi_id)
-        bangumi_payment = bangumi_data.get('payment')
-        if bangumi_payment and bangumi_payment['price'] != '0':
-            log.w("It's a paid item")
-        # ep_ids = collect_bangumi_epids(bangumi_data)
-
         frag = urllib.parse.urlparse(self.url).fragment
         if frag:
             episode_id = frag
         else:
-            episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page)
+            episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page) or re.search(r'\/ep(\d+)', self.url).group(1)
         # cont = post_content('http://bangumi.bilibili.com/web_api/get_source', post_data=dict(episode_id=episode_id))
         # cid = json.loads(cont)['result']['cid']
         cont = get_content('http://bangumi.bilibili.com/web_api/episode/{}.json'.format(episode_id))
         ep_info = json.loads(cont)['result']['currentEpisode']
 
+        bangumi_data = get_bangumi_info(str(ep_info['seasonId']))
+        bangumi_payment = bangumi_data.get('payment')
+        if bangumi_payment and bangumi_payment['price'] != '0':
+            log.w("It's a paid item")
+        # ep_ids = collect_bangumi_epids(bangumi_data)
+
         index_title = ep_info['indexTitle']
         long_title = ep_info['longTitle'].strip()
         cid = ep_info['danmaku']
@@ -295,10 +297,10 @@ def collect_bangumi_epids(json_data):
     eps = json_data['episodes'][::-1]
     return [ep['episode_id'] for ep in eps]
 
-def get_bangumi_info(bangumi_id):
+def get_bangumi_info(season_id):
     BASE_URL = 'http://bangumi.bilibili.com/jsonp/seasoninfo/'
     long_epoch = int(time.time() * 1000)
-    req_url = BASE_URL + bangumi_id + '.ver?callback=seasonListCallback&jsonp=jsonp&_=' + str(long_epoch)
+    req_url = BASE_URL + season_id + '.ver?callback=seasonListCallback&jsonp=jsonp&_=' + str(long_epoch)
     season_data = get_content(req_url)
     season_data = season_data[len('seasonListCallback('):]
     season_data = season_data[: -1 * len(');')]

From 0ed54eea3f3924f15b68bb81bbe4c7b6149e21c8 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Mon, 12 Feb 2018 01:15:57 +0800
Subject: [PATCH 0602/1225] Parameter type error

#you-get -du https://www.douyu.com/lpl
[DEBUG] get_content: https://m.douyu.com/lpl
...
Traceback (most recent call last):
...
 File "/usr/local/lib/python3.5/dist-packages/you_get/common.py", line 1574, in any_download
    m.download(url, **kwargs)
  File "/usr/local/lib/python3.5/dist-packages/you_get/extractors/douyutv.py", line 81, in douyutv_download
    download_url_ffmpeg(real_url, title, 'flv', None, output_dir = output_dir, merge = merge)
  File "/usr/local/lib/python3.5/dist-packages/you_get/common.py", line 1013, in download_url_ffmpeg
    if params.get('-y', False):  # None or unset ->False
AttributeError: 'NoneType' object has no attribute 'get'
---
 src/you_get/extractors/douyutv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 4bd0e14cf9..b7b15e742e 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -73,7 +73,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_url_ffmpeg(real_url, title, 'flv', None, output_dir = output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', params={}, output_dir = output_dir, merge = merge)
 
 site_info = "douyu.com"
 download = douyutv_download

From 0b8731cbc743361706aab269f5c1e1a7991447d9 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Tue, 13 Feb 2018 03:46:43 +0800
Subject: [PATCH 0603/1225] Update quanmin.py

api changed, and no need of import time
---
 src/you_get/extractors/quanmin.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
index bf1af65976..7428d1cbf0 100644
--- a/src/you_get/extractors/quanmin.py
+++ b/src/you_get/extractors/quanmin.py
@@ -4,7 +4,6 @@
 
 from ..common import *
 import json
-import time
 
 def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     roomid = url.split('/')[3].split('?')[0]
@@ -17,7 +16,8 @@ def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     if not data["play_status"]:
         raise ValueError("The live stream is not online!")
-    real_url = "http://flv.quanmin.tv/live/{}.flv".format(roomid)
+        
+    real_url = data["live"]["ws"]["flv"]["5"]["src"]
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From f6be4a2904b3b1451b1c02f2f58a05b3f789d8f6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 15 Feb 2018 21:30:31 +0100
Subject: [PATCH 0604/1225] [acfun] update API

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index a5ea765920..c521422f60 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -49,7 +49,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     """
 
     #first call the main parasing API
-    info = json.loads(get_content('http://www.acfun.tv/video/getVideo.aspx?id=' + vid))
+    info = json.loads(get_content('http://www.acfun.cn/video/getVideo.aspx?id=' + vid))
 
     sourceType = info['sourceType']
 

From 739504e988541216ebfc6529b12ed1d3300807ee Mon Sep 17 00:00:00 2001
From: Kugelblitz <21170940+Kugel-Blitz@users.noreply.github.com>
Date: Sun, 18 Feb 2018 11:08:17 +1300
Subject: [PATCH 0605/1225] Change code to 0507

`mp4hd3` format of some videos cannot be extracted with 0513, but can be extracted with 0507.
---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 9d74b9c88c..099552cff1 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0513'
+        self.ccode = '0507'
         self.utid = None
 
     def youku_ups(self):

From 3ce35a86f089f15ef5df88c494ffd75a62c256a4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 18 Feb 2018 00:38:00 +0100
Subject: [PATCH 0606/1225] [yixia] make things right

---
 src/you_get/extractors/yixia.py | 34 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index a37c359c7a..ccaaf5462b 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -11,14 +11,14 @@
 def yixia_miaopai_download_by_scid(scid, output_dir = '.', merge = True, info_only = False):
     """"""
     api_endpoint = 'http://api.miaopai.com/m/v2_channel.json?fillType=259&scid={scid}&vend=miaopai'.format(scid = scid)
-    
+
     html = get_content(api_endpoint)
-    
+
     api_content = loads(html)
-    
+
     video_url = match1(api_content['result']['stream']['base'], r'(.+)\?vend')
     title = api_content['result']['ext']['t']
-    
+
     type, ext, size = url_info(video_url)
 
     print_info(site_info, title, type, size)
@@ -29,14 +29,14 @@ def yixia_miaopai_download_by_scid(scid, output_dir = '.', merge = True, info_on
 def yixia_xiaokaxiu_download_by_scid(scid, output_dir = '.', merge = True, info_only = False):
     """"""
     api_endpoint = 'http://api.xiaokaxiu.com/video/web/get_play_video?scid={scid}'.format(scid = scid)
-    
+
     html = get_content(api_endpoint)
-    
+
     api_content = loads(html)
-    
+
     video_url = api_content['data']['linkurl']
     title = api_content['data']['title']
-    
+
     type, ext, size = url_info(video_url)
 
     print_info(site_info, title, type, size)
@@ -50,20 +50,16 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     if 'miaopai.com' in hostname:  #Miaopai
         yixia_download_by_scid = yixia_miaopai_download_by_scid
         site_info = "Yixia Miaopai"
-        
-        if re.match(r'https?://www.miaopai.com/show/channel/.+', url):  #PC
-            scid = match1(url, r'https?://www.miaopai.com/show/channel/(.+)\.htm')
-        elif re.match(r'https?://www.miaopai.com/show/.+', url):  #PC
-            scid = match1(url, r'https?://www.miaopai.com/show/(.+)\.htm')
-        elif re.match(r'https?://m.miaopai.com/show/channel/.+', url):  #Mobile
-            scid = match1(url, r'https?://m.miaopai.com/show/channel/(.+)\.htm')
-            if scid == None :
-                scid = match1(url, r'https?://m.miaopai.com/show/channel/(.+)')
+
+        scid = match1(url, r'miaopai\.com/show/channel/(.+)\.htm') or \
+               match1(url, r'miaopai\.com/show/(.+)\.htm') or \
+               match1(url, r'm\.miaopai\.com/show/channel/(.+)\.htm') or \
+               match1(url, r'm\.miaopai\.com/show/channel/(.+)')
 
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu
         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid
         site_info = "Yixia Xiaokaxiu"
-        
+
         if re.match(r'http://v.xiaokaxiu.com/v/.+\.html', url):  #PC
             scid = match1(url, r'http://v.xiaokaxiu.com/v/(.+)\.html')
         elif re.match(r'http://m.xiaokaxiu.com/m/.+\.html', url):  #Mobile
@@ -71,7 +67,7 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
 
     else:
         pass
-    
+
     yixia_download_by_scid(scid, output_dir, merge, info_only)
 
 site_info = "Yixia"

From 6e38f28474984faa070da24633c484cf98198032 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 18 Feb 2018 00:52:35 +0100
Subject: [PATCH 0607/1225] [twitter] normalize mobile URL

---
 src/you_get/extractors/twitter.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 04fd8773d1..9c5acb3127 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -15,6 +15,9 @@ def extract_m3u(source):
     return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if re.match(r'https?://mobile', url): # normalize mobile URL
+        url = 'https://' + match1(url, r'//mobile\.(.+)')
+
     html = get_html(url)
     screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)

From fb2c9368933d4ba59a622c65765c2368fd12c19d Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Mon, 19 Feb 2018 22:00:55 +0800
Subject: [PATCH 0608/1225] Add longzhu.com

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/longzhu.py  | 73 ++++++++++++++++++++++++++++++
 3 files changed, 75 insertions(+)
 create mode 100644 src/you_get/extractors/longzhu.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a4a036a478..4ea144f175 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -74,6 +74,7 @@
     'le'               : 'le',
     'letv'             : 'le',
     'lizhi'            : 'lizhi',
+	'longzhu'          : 'longzhu',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
     'mgtv'             : 'mgtv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 46e5c89c67..ec9e86ae42 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -41,6 +41,7 @@
 from .kuwo import *
 from .le import *
 from .lizhi import *
+from .longzhu import *
 from .magisto import *
 from .metacafe import *
 from .mgtv import *
diff --git a/src/you_get/extractors/longzhu.py b/src/you_get/extractors/longzhu.py
new file mode 100644
index 0000000000..ed0cb08447
--- /dev/null
+++ b/src/you_get/extractors/longzhu.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+
+__all__ = ['longzhu_download']
+
+import json
+from ..common import (
+    get_content,
+    match1,
+    print_info,
+    download_urls,
+    playlist_not_supported,
+)
+from ..common import player
+
+def longzhu_download(url, output_dir = '.', merge=True, info_only=False, **kwargs):
+    web_domain = url.split('/')[2]
+    if (web_domain == 'star.longzhu.com') or (web_domain == 'y.longzhu.com'):
+        domain = url.split('/')[3].split('?')[0]
+        m_url = 'http://m.longzhu.com/{0}'.format(domain)
+        m_html = get_content(m_url)
+        room_id_patt = r'var\s*roomId\s*=\s*(\d+);'
+        room_id = match1(m_html,room_id_patt)
+
+        json_url = 'http://liveapi.plu.cn/liveapp/roomstatus?roomId={0}'.format(room_id)
+        content = get_content(json_url)
+        data = json.loads(content)
+        streamUri = data['streamUri']
+        if len(streamUri) <= 4:
+            raise ValueError('The live stream is not online!')
+        title = data['title']
+        streamer = data['userName']
+        title = str.format(streamer,': ',title)
+
+        steam_api_url = 'http://livestream.plu.cn/live/getlivePlayurl?roomId={0}'.format(room_id)
+        content = get_content(steam_api_url)
+        data = json.loads(content)
+        isonline = data.get('isTransfer')
+        if isonline == '0':
+            raise ValueError('The live stream is not online!')
+
+        real_url = data['playLines'][0]['urls'][0]['securityUrl']
+
+        print_info(site_info, title, 'flv', float('inf'))
+
+        if not info_only:
+            download_urls([real_url], title, 'flv', None, output_dir, merge=merge)
+
+    elif web_domain == 'replay.longzhu.com':
+        videoid = match1(url, r'(\d+)$')
+        json_url = 'http://liveapi.longzhu.com/livereplay/getreplayfordisplay?videoId={0}'.format(videoid)
+        content = get_content(json_url)
+        data = json.loads(content)
+
+        username = data['userName']
+        title = data['title']
+        title = str.format(username,':',title)
+        real_url = data['videoUrl']
+
+        if player:
+            print_info('Longzhu Video', title, 'm3u8', 0)
+            download_urls([real_url], title, 'm3u8', 0, output_dir, merge=merge)
+        else:
+            urls = general_m3u8_extractor(real_url)
+            print_info('Longzhu Video', title, 'm3u8', 0)
+            if not info_only:
+                download_urls(urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs)
+
+    else:
+        raise ValueError('Wrong url or unsupported link ... {0}'.format(url))
+
+site_info = 'longzhu.com'
+download = longzhu_download
+download_playlist = playlist_not_supported('longzhu')
\ No newline at end of file

From a22dce896cd13ca398f748b93c73ea172b837df1 Mon Sep 17 00:00:00 2001
From: John128217 <35967907+John128217@users.noreply.github.com>
Date: Wed, 21 Feb 2018 14:23:21 -0800
Subject: [PATCH 0609/1225] Added an auto rename option and fixed the force
 option

I've noticed that if I am downloading two videos with same names from youtube (e.g. https://www.youtube.com/watch?v=606hmlA_nxw and https://www.youtube.com/watch?v=CLrXTnggUeY), only one of them will be saved (usually the bigger one according to the original script "os.path.getsize(output_filepath) >= total_size * 0.9"). However, I want them both while preserving their names from youtube. So I modified the source code. It looks like there are a lot of changes, but I just added an indent and everything changed.
Also, I've noticed that "force" is not working at all. I fixed that issue.
---
 src/you_get/common.py | 63 +++++++++++++++++++++++++++++--------------
 1 file changed, 43 insertions(+), 20 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a4a036a478..76cf5b97b2 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -134,6 +134,7 @@
 extractor_proxy = None
 cookies = None
 output_filename = None
+auto_rename = False
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
@@ -598,27 +599,40 @@ def url_save(
         tmp_headers['Referer'] = refer
     file_size = url_size(url, faker=faker, headers=tmp_headers)
 
-    if os.path.exists(filepath):
-        if not force and file_size == os.path.getsize(filepath):
-            if not is_part:
-                if bar:
-                    bar.done()
-                print(
-                    'Skipping {}: file already exists'.format(
-                        tr(os.path.basename(filepath))
+    continue_renameing = True
+    while continue_renameing:
+        continue_renameing = False
+        if os.path.exists(filepath):
+            if not force and file_size == os.path.getsize(filepath):
+                if not is_part:
+                    if bar:
+                        bar.done()
+                    print(
+                        'Skipping {}: file already exists'.format(
+                            tr(os.path.basename(filepath))
+                        )
                     )
-                )
+                else:
+                    if bar:
+                        bar.update_received(file_size)
+                return
             else:
-                if bar:
-                    bar.update_received(file_size)
-            return
-        else:
-            if not is_part:
-                if bar:
-                    bar.done()
-                print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
-    elif not os.path.exists(os.path.dirname(filepath)):
-        os.mkdir(os.path.dirname(filepath))
+                if not is_part:
+                    if bar:
+                        bar.done()
+                    if not force and auto_rename:
+                        path, ext = os.path.basename(filepath).rsplit('.', 1)
+                        if (re.compile(' \(\d\)').match(path[-4:]) is None):
+                            thisfile = path + ' (1).' + ext
+                        else:
+                            thisfile = path[:-2] + str(int(path[-2]) + 1) + ').' + ext 
+                        filepath = os.path.join(os.path.dirname(filepath), thisfile)
+                        print('Changing name to %s' % tr(os.path.basename(filepath)), '...')
+                        continue_renameing = True
+                        continue
+                    print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
+        elif not os.path.exists(os.path.dirname(filepath)):
+            os.mkdir(os.path.dirname(filepath))
 
     temp_filepath = filepath + '.download' if file_size != float('inf') \
         else filepath
@@ -883,7 +897,7 @@ def download_urls(
     output_filepath = os.path.join(output_dir, output_filename)
 
     if total_size:
-        if not force and os.path.exists(output_filepath) \
+        if not force and os.path.exists(output_filepath) and not auto_rename\
                 and os.path.getsize(output_filepath) >= total_size * 0.9:
             print('Skipping %s: file already exists' % output_filepath)
             print()
@@ -1370,6 +1384,10 @@ def print_version():
         '-l', '--playlist', action='store_true',
         help='Prefer to download a playlist'
     )
+    download_grp.add_argument(
+        '-a', '--auto-rename', action='store_true', default=False,
+        help='Auto rename same name different files'
+    )
 
     proxy_grp = parser.add_argument_group('Proxy options')
     proxy_grp = proxy_grp.add_mutually_exclusive_group()
@@ -1414,11 +1432,16 @@ def print_version():
     global player
     global extractor_proxy
     global output_filename
+    global auto_rename
 
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
     info_only = args.info
+    if args.force:
+        force = True
+    if args.auto_rename:
+        auto_rename = True
     if args.url:
         dry_run = True
     if args.json:

From c7290c7c2fa5bbf9c4623cf5694b742212d74df7 Mon Sep 17 00:00:00 2001
From: John128217 <35967907+John128217@users.noreply.github.com>
Date: Wed, 21 Feb 2018 21:55:28 -0800
Subject: [PATCH 0610/1225] A number bug fixed

Now if you can have more than 10 videos that have same names.
---
 src/you_get/common.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 76cf5b97b2..0d1b181045 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -622,10 +622,13 @@ def url_save(
                         bar.done()
                     if not force and auto_rename:
                         path, ext = os.path.basename(filepath).rsplit('.', 1)
-                        if (re.compile(' \(\d\)').match(path[-4:]) is None):
+                        finder = re.compile(' \([1-9]\d*?\)$')
+                        if (finder.search(path) is None):
                             thisfile = path + ' (1).' + ext
                         else:
-                            thisfile = path[:-2] + str(int(path[-2]) + 1) + ').' + ext 
+                            def numreturn(a):
+                                return ' (' + str(int(a.group()[2:-1]) + 1) + ').'
+                            thisfile = finder.sub(numreturn, path) + ext
                         filepath = os.path.join(os.path.dirname(filepath), thisfile)
                         print('Changing name to %s' % tr(os.path.basename(filepath)), '...')
                         continue_renameing = True

From b4850f5a5907a263d36d7d78e231f86e2321bd4c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 22 Feb 2018 17:21:43 +0100
Subject: [PATCH 0611/1225] [common] indent!

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 4ea144f175..b4d5784168 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -74,7 +74,7 @@
     'le'               : 'le',
     'letv'             : 'le',
     'lizhi'            : 'lizhi',
-	'longzhu'          : 'longzhu',
+    'longzhu'          : 'longzhu',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
     'mgtv'             : 'mgtv',

From 0b50fdfab430cff3b1e02c17def260ae0a5b47a3 Mon Sep 17 00:00:00 2001
From: perror <15058342792@163.com>
Date: Wed, 28 Feb 2018 16:45:48 +0800
Subject: [PATCH 0612/1225] [ixigua] fix URL request error and video download
 error and video encryption parameters acquisition

---
 src/you_get/extractors/ixigua.py | 40 ++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index aaed195dbe..0c668e82a6 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -5,30 +5,35 @@
 import binascii
 from ..common import *
 
-def get_video_id(text):
-    re_id = r"videoId: '(.*?)'"
-    return re.findall(re_id, text)[0]
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36'
+                  ' (KHTML, like Gecko) Chrome/61.0.3163.100 Mobile Safari/537.36'
+}
+
 
 def get_r():
     return str(random.random())[2:]
 
+
 def right_shift(val, n):
     return val >> n if val >= 0 else (val + 0x100000000) >> n
 
+
 def get_s(text):
     """get video info"""
-    id = get_video_id(text)
+    js_data = json.loads(text)
+    id = js_data['data']['video_id']
     p = get_r()
     url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % id
     n = parse.urlparse(url).path + '?r=%s' % p
     c = binascii.crc32(n.encode('utf-8'))
     s = right_shift(c, 0)
-    title = ''.join(re.findall(r"title: '(.*?)',", text))
-    return url + '?r=%s&s=%s' % (p, s), title
+    return url + '?r=%s&s=%s' % (p, s), js_data['data']['title']
+
 
 def get_moment(url, user_id, base_url, video_list):
     """Recursively obtaining a video list"""
-    video_list_data = json.loads(get_content(url))
+    video_list_data = json.loads(get_content(url, headers=headers))
     if not video_list_data['next']['max_behot_time']:
         return video_list
     [video_list.append(i["display_url"]) for i in video_list_data["data"]]
@@ -41,23 +46,33 @@ def get_moment(url, user_id, base_url, video_list):
     }
     return get_moment(**_param)
 
+
 def ixigua_download(url, output_dir='.', info_only=False, **kwargs):
     """ Download a single video
         Sample URL: https://www.ixigua.com/a6487187567887254029/#mid=59051127876
     """
     try:
-        video_info_url, title = get_s(get_content(url))
-        video_info = json.loads(get_content(video_info_url))
+        video_page_id = re.findall('(\d+)', [i for i in url.split('/') if i][3])[0] if 'toutiao.com' in url \
+            else re.findall('(\d+)', [i for i in url.split('/') if i][2])[0]
+
+        video_start_info_url = r'https://m.ixigua.com/i{}/info/'.format(video_page_id)
+        video_info_url, title = get_s(get_content(video_start_info_url, headers=headers or kwargs.get('headers', {})))
+        video_info = json.loads(get_content(video_info_url, headers=headers or kwargs.get('headers', {})))
     except Exception:
         raise NotImplementedError(url)
     try:
         video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"]).decode()
     except Exception:
         raise NotImplementedError(url)
-    filetype, ext, size = url_info(video_url)
+    filetype, ext, size = url_info(video_url, headers=headers or kwargs.get('headers', {}))
     print_info(site_info, title, filetype, size)
     if not info_only:
-        download_urls([video_url], title, ext, size, output_dir=output_dir)
+        _param = {
+            'output_dir': output_dir,
+            'headers': headers or kwargs.get('headers', {})
+        }
+        download_urls([video_url], title, ext, size, **_param)
+
 
 def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs):
     """Download all video from the user's video list
@@ -80,6 +95,7 @@ def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs):
     for i in get_moment(**_param):
         ixigua_download(i, output_dir, info_only, **kwargs)
 
+
 site_info = "ixigua.com"
 download = ixigua_download
-download_playlist = ixigua_download_playlist
\ No newline at end of file
+download_playlist = ixigua_download_playlist

From 7633898850f6ed30c78e1fb5bdb0f96b81d9d87a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 1 Mar 2018 22:55:09 +0100
Subject: [PATCH 0613/1225] version 0.4.1040

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 2d4ff9d03a..7e220d0dfe 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1025'
+__version__ = '0.4.1040'

From 1900f7608cc2756d5460c99eb792c8e0eb42e7f4 Mon Sep 17 00:00:00 2001
From: mq-liu <mingquan_liu@163.com>
Date: Wed, 7 Mar 2018 09:48:11 +0800
Subject: [PATCH 0614/1225] fix bilibili download fail

the bilibili api has changed
"https://interface.bilibili.com/v2/playurl?cid=33250486&appkey=84956560bc028eb7&otype=json&type=&quality=0&qn=0&sign=a1b0401c8bf70d676bab133fa032469f"
---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e5abccab6b..046d2cb188 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -23,7 +23,7 @@
 class Bilibili(VideoExtractor):
     name = 'Bilibili'
     live_api = 'http://live.bilibili.com/api/playurl?cid={}&otype=json'
-    api_url = 'http://interface.bilibili.com/playurl?'
+    api_url = 'http://interface.bilibili.com/v2/playurl?'
     bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
     live_room_init_api_url = 'https://api.live.bilibili.com/room/v1/Room/room_init?id={}'
     live_room_info_api_url = 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id={}'

From 92eb72bc7d20370e2835ed78dad94c0accaa068a Mon Sep 17 00:00:00 2001
From: Kugel-Blitz <21170940+Kugel-Blitz@users.noreply.github.com>
Date: Sun, 11 Mar 2018 10:06:19 +1300
Subject: [PATCH 0615/1225] Use 0513 when cookies are used

0507 doesn't seem to honour cookies when they're loaded.
---
 src/you_get/extractors/youku.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 9d74b9c88c..d40d16c194 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,10 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0513'
+        if cookies:
+            self.ccode = '0513'
+        else:
+            self.ccode = '0507'
         self.utid = None
 
     def youku_ups(self):

From 344502af0d223def2a9ed0fde3766f6f7490b23b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 15 Mar 2018 22:38:23 +0100
Subject: [PATCH 0616/1225] [youku] resolve conflict

---
 src/you_get/extractors/youku.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 099552cff1..d40d16c194 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,10 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0507'
+        if cookies:
+            self.ccode = '0513'
+        else:
+            self.ccode = '0507'
         self.utid = None
 
     def youku_ups(self):

From fdf53508388135917bb976319b2be01b96034634 Mon Sep 17 00:00:00 2001
From: Phun <aaaphun@gmail.com>
Date: Mon, 19 Mar 2018 12:48:38 +0800
Subject: [PATCH 0617/1225] fix the bug of v.qq.com

---
 src/you_get/extractors/qq.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 89dd7b61c6..5591e3eb42 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -122,9 +122,9 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         return
 
     #do redirect
-    if 'v.qq.com/page' in url:
+    if 'v.qq.com/x' in url:
         # for URLs like this:
-        # http://v.qq.com/page/k/9/7/k0194pwgw97.html
+        # https://v.qq.com/x/page/r05533mns3s.html
         new_url = url_locations([url])[0]
         if url == new_url:
             #redirect in js?

From 3faaebb6762ff1bbea1e9b45b6dd348a92ddbcfc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 22 Mar 2018 22:40:07 +0100
Subject: [PATCH 0618/1225] [qq] no more redirect (close #2586)

---
 src/you_get/extractors/qq.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 5591e3eb42..ffca5a8571 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -121,18 +121,6 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             qq_download_by_vid(vid, vid, output_dir, merge, info_only)
         return
 
-    #do redirect
-    if 'v.qq.com/x' in url:
-        # for URLs like this:
-        # https://v.qq.com/x/page/r05533mns3s.html
-        new_url = url_locations([url])[0]
-        if url == new_url:
-            #redirect in js?
-            content = get_content(url)
-            url = match1(content,r'window\.location\.href="(.*?)"')
-        else:
-            url = new_url
-
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         content = get_content(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')

From 8979cd63eaa0979f249e4132a10b3706c4952b02 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 22 Mar 2018 22:44:33 +0100
Subject: [PATCH 0619/1225] [qq] break if no pay

---
 src/you_get/extractors/qq.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index ffca5a8571..7b1a6860f3 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -47,6 +47,9 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
             else:
                 log.w(key_json['msg'])
             break
+        if key_json.get('filename') is None:
+            log.w(key_json['msg'])
+            break
 
         part_urls.append(url)
         _, ext, size = url_info(url)

From 821e639e025296b4c041d0535ca4d95ad72ea397 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 22 Mar 2018 22:46:12 +0100
Subject: [PATCH 0620/1225] [youku] boom boom boom

---
 src/you_get/extractors/youku.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d40d16c194..bc4d808829 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,10 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        if cookies:
-            self.ccode = '0513'
-        else:
-            self.ccode = '0507'
+        self.ccode = '0590'
         self.utid = None
 
     def youku_ups(self):

From d3719ed4b62be2697e18755bcda7cb2249c8d7c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 24 Mar 2018 23:56:38 +0100
Subject: [PATCH 0621/1225] [bilibili] warn when target URL is a playlist

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 046d2cb188..7e5bdb37da 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -167,8 +167,8 @@ def entry(self, **kwargs):
             qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
             return
 
-        has_plist = re.search(r'<option', self.page)
-        if has_plist and r1('index_(\d+).html', self.url) is None:
+        has_plist = re.search(r'"page":2', self.page)
+        if has_plist:
             log.w('This page contains a playlist. (use --playlist to download all videos.)')
 
         try:

From 6e5e385620a527bd006c2220e813b9c7aa0acffb Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Mon, 26 Mar 2018 12:05:12 +0200
Subject: [PATCH 0622/1225] from ..common import general_m3u8_extractor

Import the definition of __general_m3u8_extractor()__ for the function call on line 64...

flake8 testing of https://github.com/soimort/you-get on Python 3.6.3

$ __flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics__
```
./src/you_get/extractors/longzhu.py:63:20: F821 undefined name 'general_m3u8_extractor'
            urls = general_m3u8_extractor(real_url)
                   ^
1     F821 undefined name 'general_m3u8_extractor'
```
---
 src/you_get/extractors/longzhu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/longzhu.py b/src/you_get/extractors/longzhu.py
index ed0cb08447..29b340c591 100644
--- a/src/you_get/extractors/longzhu.py
+++ b/src/you_get/extractors/longzhu.py
@@ -5,6 +5,7 @@
 import json
 from ..common import (
     get_content,
+    general_m3u8_extractor,
     match1,
     print_info,
     download_urls,
@@ -70,4 +71,4 @@ def longzhu_download(url, output_dir = '.', merge=True, info_only=False, **kwarg
 
 site_info = 'longzhu.com'
 download = longzhu_download
-download_playlist = playlist_not_supported('longzhu')
\ No newline at end of file
+download_playlist = playlist_not_supported('longzhu')

From 43923bc8f6c7df552e672a4e80aed0e58010964d Mon Sep 17 00:00:00 2001
From: JayXon <jayxon@gmail.com>
Date: Thu, 29 Mar 2018 00:59:28 -0700
Subject: [PATCH 0623/1225] [youku] use default ckey

1080p works
---
 src/you_get/extractors/youku.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index bc4d808829..fc1a5cd2f1 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,10 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0590'
+        self.ccode = '0502'
+        # Found in http://g.alicdn.com/player/ykplayer/0.5.28/youku-player.min.js
+        # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
+        self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'
         self.utid = None
 
     def youku_ups(self):
@@ -86,6 +89,7 @@ def youku_ups(self):
         url += '&client_ip=192.168.1.1'
         url += '&utid=' + self.utid
         url += '&client_ts=' + str(int(time.time()))
+        url += '&ckey=' + urllib.parse.quote(self.ckey)
         if self.password_protected:
             url += '&password=' + self.password
         headers = dict(Referer=self.referer)

From 6a9039aab110f40ba6a4fed5915d58cffee8aa46 Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Wed, 11 Apr 2018 19:59:14 +0800
Subject: [PATCH 0624/1225] fix https://v.qq.com/x/page/d0552xbadkl.html
 https://y.qq.com/n/yqq/mv/v/g00268vlkzy.html

---
 src/you_get/extractors/qq.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 6a859b8bc1..915f1b4bc5 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -140,7 +140,8 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = ""
         if rurl:
             vid = rurl.split('/')[-1].split('.')[0]
-            if vid == "undefined":
+            # https://v.qq.com/x/page/d0552xbadkl.html https://y.qq.com/n/yqq/mv/v/g00268vlkzy.html
+            if vid == "undefined" or vid == "index":
                 vid = ""
         vid = vid if vid else url.split('/')[-1].split('.')[0] #https://v.qq.com/x/cover/ps6mnfqyrfo7es3/q0181hpdvo5.html?
         vid = vid if vid else match1(content, r'vid"*\s*:\s*"\s*([^"]+)"') #general fallback

From ead0979ca133e75b62835c3d3ac2783955534a0f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 12 Apr 2018 03:19:12 +0200
Subject: [PATCH 0625/1225] [universal] relative path lah

---
 src/you_get/extractors/universal.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 633cf55b4b..b6bb68b1b5 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -67,6 +67,14 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         urls += re.findall(r'href="(https?://[^"]+\.png)"', page, re.I)
         urls += re.findall(r'href="(https?://[^"]+\.gif)"', page, re.I)
 
+        # relative path
+        rel_urls = []
+        rel_urls += re.findall(r'href="(\.[^"]+\.jpe?g)"', page, re.I)
+        rel_urls += re.findall(r'href="(\.[^"]+\.png)"', page, re.I)
+        rel_urls += re.findall(r'href="(\.[^"]+\.gif)"', page, re.I)
+        for rel_url in rel_urls:
+            urls += [ r1(r'(.*/)', url) + rel_url ]
+
         # MPEG-DASH MPD
         mpd_urls = re.findall(r'src="(https?://[^"]+\.mpd)"', page)
         for mpd_url in mpd_urls:

From 25aa2ac2e5f5b408edfc53f64a6706a716f0e0c3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Apr 2018 16:34:15 +0200
Subject: [PATCH 0626/1225] [universal] better extraction of title and ext

---
 src/you_get/extractors/universal.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index b6bb68b1b5..e343d4cdfb 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -112,10 +112,9 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
     else:
         # direct download
-        filename = parse.unquote(url.split('/')[-1])
-        title = '.'.join(filename.split('.')[:-1])
-        ext = filename.split('.')[-1]
-        _, _, size = url_info(url, faker=True)
+        filename = parse.unquote(url.split('/')[-1]) or parse.unquote(url.split('/')[-2])
+        title = '.'.join(filename.split('.')[:-1]) or filename
+        _, ext, size = url_info(url, faker=True)
         print_info(site_info, title, ext, size)
         if not info_only:
             download_urls([url], title, ext, size,

From c77f29861c27725811c54285f351fc120279d75c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Apr 2018 17:07:46 +0200
Subject: [PATCH 0627/1225] [universal] support Open Graph og:video:url

---
 src/you_get/extractors/universal.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index e343d4cdfb..573d8eea4e 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -31,6 +31,19 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if page_title:
             page_title = unescape_html(page_title)
 
+        meta_videos = re.findall(r'<meta property="og:video:url" content="([^"]*)"', page)
+        if meta_videos:
+            for meta_video in meta_videos:
+                meta_video_url = unescape_html(meta_video)
+                type_, ext, size = url_info(meta_video_url)
+                print_info(site_info, page_title, type_, size)
+                if not info_only:
+                    download_urls([meta_video_url], page_title,
+                                  ext, size,
+                                  output_dir=output_dir, merge=merge,
+                                  faker=True)
+            return
+
         hls_urls = re.findall(r'(https?://[^;"\'\\]+' + '\.m3u8?' +
                               r'[^;"\'\\]*)', page)
         if hls_urls:

From 029272d1406aba23efe1c2be5ca1d579e5912d5c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Apr 2018 17:13:08 +0200
Subject: [PATCH 0628/1225] version 0.4.1060

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 7e220d0dfe..4d91c55d38 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1040'
+__version__ = '0.4.1060'

From b28d78f71d50369cb6d306ef3e68430dedf86f1a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 19 Apr 2018 03:22:18 +0200
Subject: [PATCH 0629/1225] [twitter] support twitter moments

---
 src/you_get/extractors/twitter.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 9c5acb3127..894439aa45 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -18,6 +18,17 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     if re.match(r'https?://mobile', url): # normalize mobile URL
         url = 'https://' + match1(url, r'//mobile\.(.+)')
 
+    if re.match(r'https?://twitter\.com/i/moments/', url): # moments
+        html = get_html(url)
+        paths = re.findall(r'data-permalink-path="([^"]+)"', html)
+        for path in paths:
+            twitter_download('https://twitter.com' + path,
+                             output_dir=output_dir,
+                             merge=merge,
+                             info_only=info_only,
+                             **kwargs)
+        return
+
     html = get_html(url)
     screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
@@ -58,7 +69,10 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             url = r1(r'<meta\s*property="og:video:url"\s*content="([^"]+)"', html)
             if not url:
                 url = 'https://twitter.com/i/videos/%s' % item_id
-            html = get_content(url)
+            try:
+                html = get_content(url)
+            except:
+                return
 
         data_config = r1(r'data-config="([^"]*)"', html) or \
             r1(r'data-player-config="([^"]*)"', html)

From a3b15279b6c671953a48486b2000d9ff11e23806 Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Sun, 22 Apr 2018 10:38:40 +0800
Subject: [PATCH 0630/1225] fix bilibili

update xml-url to get all format of video
---
 src/you_get/extractors/bilibili.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 7e5bdb37da..d23bbe5c87 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -28,7 +28,8 @@ class Bilibili(VideoExtractor):
     live_room_init_api_url = 'https://api.live.bilibili.com/room/v1/Room/room_init?id={}'
     live_room_info_api_url = 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id={}'
 
-    SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
+    #SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
+    SEC1 = '94aba54af9065f71de72f5508f1cd42e'
     SEC2 = '9b288147e5474dd2aa67085f716c560d'
     stream_types = [
             {'id': 'hdflv'},
@@ -44,7 +45,7 @@ class Bilibili(VideoExtractor):
     @staticmethod
     def bilibili_stream_type(urls):
         url = urls[0]
-        if 'hd.flv' in url or '-112.flv' in url:
+        if 'hd.flv' in url or '-80.flv' in url:
             return 'hdflv', 'flv'
         if '-64.flv' in url:
             return 'flv720', 'flv'
@@ -59,7 +60,8 @@ def bilibili_stream_type(urls):
     def api_req(self, cid, quality, bangumi, bangumi_movie=False, **kwargs):
         ts = str(int(time.time()))
         if not bangumi:
-            params_str = 'cid={}&player=1&quality={}&ts={}'.format(cid, quality, ts)
+            #params_str = 'cid={}&player=1&quality={}&ts={}'.format(cid, quality, ts)
+            params_str = 'appkey=84956560bc028eb7&cid={}&otype=xml&qn={}&quality={}&type='.format(cid, quality, quality)
             chksum = hashlib.md5(bytes(params_str+self.SEC1, 'utf8')).hexdigest()
             api_url = self.api_url + params_str + '&sign=' + chksum
         else:
@@ -97,7 +99,7 @@ def download_by_vid(self, cid, bangumi, **kwargs):
             quality = 'hdflv' if bangumi else 'flv'
 
         info_only = kwargs.get('info_only')
-        for qlt in range(4, -1, -1):
+        for qlt in [116,112,80,74,64,32,16,15]:
             api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
             self.parse_bili_xml(api_xml)
         if not info_only or stream_id:

From d057a49e5b4222cb69b47008cb9e0af1b6b0209f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 24 Apr 2018 14:46:38 +0200
Subject: [PATCH 0631/1225] [common] url_to_module: quote non-ASCII characters
 in URL

---
 src/you_get/common.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 11200d10eb..1a6cac2bce 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1581,6 +1581,9 @@ def url_to_module(url):
     domain = r1(r'(\.[^.]+\.[^.]+)$', video_host) or video_host
     assert domain, 'unsupported url: ' + url
 
+    # all non-ASCII code points must be quoted (percent-encoded UTF-8)
+    url = ''.join([ch if ord(ch) in range(128) else parse.quote(ch) for ch in url])
+
     k = r1(r'([^.]+)', domain)
     if k in SITES:
         return (

From ff6deaf2bde4a8e81094c7ff5893fa4d9b30efb0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 24 Apr 2018 14:48:20 +0200
Subject: [PATCH 0632/1225] [tumblr] fallback to universal_download

---
 src/you_get/extractors/tumblr.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 5817e5483c..fe4973bed0 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -70,6 +70,11 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         real_url = r1(r'<source src="([^"]*)"', html)
     if not real_url:
         iframe_url = r1(r'<[^>]+tumblr_video_container[^>]+><iframe[^>]+src=[\'"]([^\'"]*)[\'"]', html)
+
+        if iframe_url is None:
+            universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
+            return
+
         if iframe_url:
             iframe_html = get_content(iframe_url, headers=fake_headers)
             real_url = r1(r'<video[^>]*>[\n ]*<source[^>]+src=[\'"]([^\'"]*)[\'"]', iframe_html)

From 44960677c4c315e479d3b2015582f98f32d40c48 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Apr 2018 22:30:46 +0200
Subject: [PATCH 0633/1225] [common] use quoted video_host and video_url as
 well

---
 src/you_get/common.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1a6cac2bce..e30008549c 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1583,6 +1583,8 @@ def url_to_module(url):
 
     # all non-ASCII code points must be quoted (percent-encoded UTF-8)
     url = ''.join([ch if ord(ch) in range(128) else parse.quote(ch) for ch in url])
+    video_host = r1(r'https?://([^/]+)/', url)
+    video_url = r1(r'https?://[^/]+(.*)', url)
 
     k = r1(r'([^.]+)', domain)
     if k in SITES:

From fe34688d07872e18fa0127c969a4f05152e3342a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Apr 2018 22:42:23 +0200
Subject: [PATCH 0634/1225] [universal] fix my brain damage since 2015
 (2c7aa3b)

---
 src/you_get/extractors/universal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 573d8eea4e..57994b9c3f 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -112,7 +112,7 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         for candy in candies:
             try:
                 mime, ext, size = url_info(candy['url'], faker=True)
-                if not size: size = float('Int')
+                if not size: size = float('Inf')
             except:
                 continue
             else:

From bcc98c5a5cfae4cd13487f0a51662ede35e746bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Apr 2018 22:59:39 +0200
Subject: [PATCH 0635/1225] [universal] use faker only if necessary

---
 src/you_get/extractors/universal.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 57994b9c3f..6a1c2d302f 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -111,16 +111,25 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         for candy in candies:
             try:
-                mime, ext, size = url_info(candy['url'], faker=True)
-                if not size: size = float('Inf')
+                try:
+                    mime, ext, size = url_info(candy['url'], faker=False)
+                    assert size
+                except:
+                    mime, ext, size = url_info(candy['url'], faker=True)
+                    if not size: size = float('Inf')
             except:
                 continue
             else:
                 print_info(site_info, candy['title'], ext, size)
                 if not info_only:
-                    download_urls([candy['url']], candy['title'], ext, size,
-                                  output_dir=output_dir, merge=merge,
-                                  faker=True)
+                    try:
+                        download_urls([candy['url']], candy['title'], ext, size,
+                                      output_dir=output_dir, merge=merge,
+                                      faker=False)
+                    except:
+                        download_urls([candy['url']], candy['title'], ext, size,
+                                      output_dir=output_dir, merge=merge,
+                                      faker=True)
         return
 
     else:

From 4f1b609d71a04672a2b95b4fb13f0ba486e57df0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 Apr 2018 18:59:52 +0200
Subject: [PATCH 0636/1225] [bilibili] fix title

---
 src/you_get/extractors/bilibili.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index d23bbe5c87..916782af92 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -130,6 +130,9 @@ def prepare(self, **kwargs):
         m = re.search(r'<h1.*?>(.*?)</h1>', self.page) or re.search(r'<h1 title="([^"]+)">', self.page)
         if m is not None:
             self.title = m.group(1)
+            s = re.search(r'<span>([^<]+)</span>', m.group(1))
+            if s:
+                self.title = unescape_html(s.group(1))
         if self.title is None:
             m = re.search(r'property="og:title" content="([^"]+)"', self.page)
             if m is not None:

From 18d3cf0eb424fa92473141c2af6a9d0183550a72 Mon Sep 17 00:00:00 2001
From: QYLGithub <15058342792@163.com>
Date: Sun, 29 Apr 2018 11:38:49 +0800
Subject: [PATCH 0637/1225] Call toutiao.py method

---
 src/you_get/extractors/ixigua.py | 98 ++------------------------------
 1 file changed, 5 insertions(+), 93 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 0c668e82a6..bc19b1d0cd 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -1,101 +1,13 @@
 #!/usr/bin/env python
 __all__ = ['ixigua_download', 'ixigua_download_playlist']
-import base64
-import random
-import binascii
-from ..common import *
+from .toutiao import download as toutiao_download
+from .toutiao import download_playlist as toutiao_download_playlist
 
-headers = {
-    'User-Agent': 'Mozilla/5.0 (Linux; Android 5.0; SM-G900P Build/LRX21T) AppleWebKit/537.36'
-                  ' (KHTML, like Gecko) Chrome/61.0.3163.100 Mobile Safari/537.36'
-}
 
-
-def get_r():
-    return str(random.random())[2:]
-
-
-def right_shift(val, n):
-    return val >> n if val >= 0 else (val + 0x100000000) >> n
-
-
-def get_s(text):
-    """get video info"""
-    js_data = json.loads(text)
-    id = js_data['data']['video_id']
-    p = get_r()
-    url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % id
-    n = parse.urlparse(url).path + '?r=%s' % p
-    c = binascii.crc32(n.encode('utf-8'))
-    s = right_shift(c, 0)
-    return url + '?r=%s&s=%s' % (p, s), js_data['data']['title']
-
-
-def get_moment(url, user_id, base_url, video_list):
-    """Recursively obtaining a video list"""
-    video_list_data = json.loads(get_content(url, headers=headers))
-    if not video_list_data['next']['max_behot_time']:
-        return video_list
-    [video_list.append(i["display_url"]) for i in video_list_data["data"]]
-    max_behot_time = video_list_data['next']['max_behot_time']
-    _param = {
-        'user_id': user_id,
-        'base_url': base_url,
-        'video_list': video_list,
-        'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
-    }
-    return get_moment(**_param)
-
-
-def ixigua_download(url, output_dir='.', info_only=False, **kwargs):
-    """ Download a single video
-        Sample URL: https://www.ixigua.com/a6487187567887254029/#mid=59051127876
-    """
-    try:
-        video_page_id = re.findall('(\d+)', [i for i in url.split('/') if i][3])[0] if 'toutiao.com' in url \
-            else re.findall('(\d+)', [i for i in url.split('/') if i][2])[0]
-
-        video_start_info_url = r'https://m.ixigua.com/i{}/info/'.format(video_page_id)
-        video_info_url, title = get_s(get_content(video_start_info_url, headers=headers or kwargs.get('headers', {})))
-        video_info = json.loads(get_content(video_info_url, headers=headers or kwargs.get('headers', {})))
-    except Exception:
-        raise NotImplementedError(url)
-    try:
-        video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"]).decode()
-    except Exception:
-        raise NotImplementedError(url)
-    filetype, ext, size = url_info(video_url, headers=headers or kwargs.get('headers', {}))
-    print_info(site_info, title, filetype, size)
-    if not info_only:
-        _param = {
-            'output_dir': output_dir,
-            'headers': headers or kwargs.get('headers', {})
-        }
-        download_urls([video_url], title, ext, size, **_param)
-
-
-def ixigua_download_playlist(url, output_dir='.', info_only=False, **kwargs):
-    """Download all video from the user's video list
-        Sample URL: https://www.ixigua.com/c/user/71141690831/
-    """
-    if 'user' not in url:
-        raise NotImplementedError(url)
-    user_id = url.split('/')[-2]
-    max_behot_time = 0
-    if not user_id:
-        raise NotImplementedError(url)
-    base_url = "https://www.ixigua.com/c/user/article/?user_id={user_id}" \
-               "&max_behot_time={max_behot_time}&max_repin_time=0&count=20&page_type=0"
-    _param = {
-        'user_id': user_id,
-        'base_url': base_url,
-        'video_list': [],
-        'url': base_url.format(user_id=user_id, max_behot_time=max_behot_time),
-    }
-    for i in get_moment(**_param):
-        ixigua_download(i, output_dir, info_only, **kwargs)
+def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    return toutiao_download(url.replace('ixigua', '365yg'))
 
 
 site_info = "ixigua.com"
 download = ixigua_download
-download_playlist = ixigua_download_playlist
+download_playlist = toutiao_download_playlist
\ No newline at end of file

From 351173ba797ad1ebc830ed1de223f48c3570248e Mon Sep 17 00:00:00 2001
From: yangxiaochen <yangxiaochen@58ganji.com>
Date: Thu, 10 May 2018 20:32:59 +0800
Subject: [PATCH 0638/1225] [qq] fix some error cases("check vid&filename
 failed" and "format invalid")

---
 src/you_get/extractors/qq.py | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 915f1b4bc5..15116b0c73 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -18,11 +18,14 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
     streams = video_json['fl']['fi']
     seg_cnt = video_json['vl']['vi'][0]['cl']['fc']
+    filename = video_json['vl']['vi'][0]['fn']
     if seg_cnt == 0:
         seg_cnt = 1
+    else:
+        fn_pre, magic_str, video_type = filename.split('.')
 
     best_quality = streams[-1]['name']
-    part_format_id = streams[-1]['id']
+    #part_format_id = streams[-1]['id']
 
     part_urls= []
     total_size = 0
@@ -31,7 +34,17 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         #    filename = fn_pre + '.mp4'
         #else:
         #    filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
-        filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+        #filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
+
+        # fix some error cases("check vid&filename failed" and "format invalid")
+        # https://v.qq.com/x/page/q06058th9ll.html
+        # https://v.qq.com/x/page/t060789a21e.html
+        if seg_cnt == 1:
+            part_format_id = video_json['vl']['vi'][0]['cl']['keyid'].split('.')[-1]
+        else:
+            part_format_id = video_json['vl']['vi'][0]['cl']['ci'][part - 1]['keyid'].split('.')[1]
+            filename = '.'.join([fn_pre, magic_str, str(part), video_type])
+
         key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
         part_info = get_content(key_api)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])

From 24578efa1e34c24c7828d82cc27c70b478e6740a Mon Sep 17 00:00:00 2001
From: perror <15058342792@163.com>
Date: Fri, 11 May 2018 12:01:31 +0800
Subject: [PATCH 0639/1225] repair douyutv 403 error

---
 src/you_get/extractors/douyutv.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index b7b15e742e..72a41a0ad4 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -9,6 +9,10 @@
 import time
 import re
 
+headers = {
+        'user-agent': 'Mozilla/5.0 (iPad; CPU OS 8_1_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B466 Safari/600.1.4'
+    }
+
 def douyutv_video_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     ep = 'http://vmobile.douyu.com/video/getInfo?vid='
     patt = r'show/([0-9A-Za-z]+)'
@@ -19,7 +23,7 @@ def douyutv_video_download(url, output_dir='.', merge=True, info_only=False, **k
         log.wtf('Unknown url pattern')
     vid = hit.group(1)
 
-    page = get_content(url)
+    page = get_content(url, headers=headers)
     hit = re.search(title_patt, page)
     if hit is None:
         title = vid
@@ -35,21 +39,18 @@ def douyutv_video_download(url, output_dir='.', merge=True, info_only=False, **k
         urls = general_m3u8_extractor(m3u8_url)
         download_urls(urls, title, 'ts', 0, output_dir=output_dir, merge=merge, **kwargs)
 
-def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+
+def douyutv_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if 'v.douyu.com/show/' in url:
         douyutv_video_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return
 
-    headers = {
-        'user-agent': 'Mozilla/5.0 (iPad; CPU OS 8_1_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B466 Safari/600.1.4'
-    }
-    
-    url = re.sub(r'[w.]*douyu.com','m.douyu.com',url)
+    url = re.sub(r'[w.]*douyu.com', 'm.douyu.com', url)
     html = get_content(url, headers)
     room_id_patt = r'room_id\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
     if room_id == "0":
-        room_id = url[url.rfind('/')+1:]
+        room_id = url[url.rfind('/') + 1:]
 
     api_url = "http://www.douyutv.com/api/v1/"
     args = "room/%s?aid=wp&client_sys=wp&time=%d" % (room_id, int(time.time()))
@@ -60,7 +61,7 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
     content = get_content(json_request_url, headers)
     json_content = json.loads(content)
     data = json_content['data']
-    server_status = json_content.get('error',0)
+    server_status = json_content.get('error', 0)
     if server_status is not 0:
         raise ValueError("Server returned error:%s" % server_status)
 
@@ -73,7 +74,8 @@ def douyutv_download(url, output_dir = '.', merge = True, info_only = False, **k
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
-        download_url_ffmpeg(real_url, title, 'flv', params={}, output_dir = output_dir, merge = merge)
+        download_url_ffmpeg(real_url, title, 'flv', params={}, output_dir=output_dir, merge=merge)
+
 
 site_info = "douyu.com"
 download = douyutv_download

From 50bba5527b52121a1f41b75d763a38fe2432e73e Mon Sep 17 00:00:00 2001
From: yangxiaochen <yangxiaochen@58ganji.com>
Date: Fri, 11 May 2018 17:12:01 +0800
Subject: [PATCH 0640/1225] [douyin] send the request without fake headers, the
 douyin website will return fake body or the 403 response!

---
 src/you_get/extractors/douyin.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 213837e772..20ce0e2682 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -7,6 +7,7 @@
     url_size,
     print_info,
     get_content,
+    fake_headers,
     download_urls,
     playlist_not_supported,
 )
@@ -16,13 +17,13 @@
 
 
 def douyin_download_by_url(url, **kwargs):
-    page_content = get_content(url)
+    page_content = get_content(url, headers=fake_headers)
     match_rule = re.compile(r'var data = \[(.*?)\];')
     video_info = json.loads(match_rule.findall(page_content)[0])
     video_url = video_info['video']['play_addr']['url_list'][0]
     title = video_info['cha_list'][0]['cha_name']
     video_format = 'mp4'
-    size = url_size(video_url)
+    size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,
         type=video_format, size=size
@@ -30,6 +31,7 @@ def douyin_download_by_url(url, **kwargs):
     if not kwargs['info_only']:
         download_urls(
             urls=[video_url], title=title, ext=video_format, total_size=size,
+            faker=True,
             **kwargs
         )
 

From daf630e9d782c53878b77b33a891d8003e747a72 Mon Sep 17 00:00:00 2001
From: yangxiaochen <yangxiaochen@58ganji.com>
Date: Fri, 11 May 2018 18:08:23 +0800
Subject: [PATCH 0641/1225] [douyin] fix if there is not title, you-get cannot
 works well https://www.douyin.com/share/video/6553248251821165832

---
 src/you_get/extractors/douyin.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 20ce0e2682..e39775f44f 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -21,7 +21,13 @@ def douyin_download_by_url(url, **kwargs):
     match_rule = re.compile(r'var data = \[(.*?)\];')
     video_info = json.loads(match_rule.findall(page_content)[0])
     video_url = video_info['video']['play_addr']['url_list'][0]
-    title = video_info['cha_list'][0]['cha_name']
+    # fix: https://www.douyin.com/share/video/6553248251821165832
+    # if there is no title, use desc
+    cha_list = video_info['cha_list']
+    if cha_list:
+        title = cha_list[0]['cha_name']
+    else:
+        title = video_info['desc']
     video_format = 'mp4'
     size = url_size(video_url, faker=True)
     print_info(

From 9b03331589e645d76b28fd9021b6d17426186695 Mon Sep 17 00:00:00 2001
From: Zheng Luo <rogerfederer03@gmail.com>
Date: Sat, 12 May 2018 20:08:13 -0400
Subject: [PATCH 0642/1225] Update cccode to 0510 for youku.py

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index fc1a5cd2f1..bfdb014fad 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0502'
+        self.ccode = '0510'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.28/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 9ae300029f5de925be4e1de304e2809ec694d668 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 15 May 2018 21:22:51 +0200
Subject: [PATCH 0643/1225] version 0.4.1077

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 4d91c55d38..64ef890f8b 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1060'
+__version__ = '0.4.1077'

From f7179968f8147e304fb61b26e381f28d4da07dff Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@bluewin.ch>
Date: Fri, 18 May 2018 09:08:13 +0200
Subject: [PATCH 0644/1225] Add flake8 to the testing (again)

Another attempt at #2145
---
 .travis.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 2d780e8181..ed1531b9bb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,6 +8,8 @@ python:
   - "3.6"
   - "nightly"
   - "pypy3"
+before_install: pip install flake8
+before_script: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
 script: make test
 sudo: false
 notifications:

From 59f544665ff89a270c7c1e11f90f423c7690929c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 20 May 2018 13:00:12 +0200
Subject: [PATCH 0645/1225] [ixigua] remove undefined name (#2599)

---
 src/you_get/extractors/ixigua.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index bc19b1d0cd..5913344292 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python
-__all__ = ['ixigua_download', 'ixigua_download_playlist']
+__all__ = ['ixigua_download']
+
 from .toutiao import download as toutiao_download
 from .toutiao import download_playlist as toutiao_download_playlist
 
@@ -10,4 +11,4 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
 site_info = "ixigua.com"
 download = ixigua_download
-download_playlist = toutiao_download_playlist
\ No newline at end of file
+download_playlist = toutiao_download_playlist

From 582d89e2f268ab1a72d6b065694760097c270702 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 20 May 2018 13:47:28 +0200
Subject: [PATCH 0646/1225] .travis.yml: skip flake8 on python 3.2

---
 .travis.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index ed1531b9bb..9df327b0e4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,7 +9,8 @@ python:
   - "nightly"
   - "pypy3"
 before_install: pip install flake8
-before_script: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics
+before_script:
+  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi
 script: make test
 sudo: false
 notifications:

From 25b1c25517fbfb71a9e997edb4dad991249da6f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E9=9F=A9=E6=9E=97=E6=B3=BD?=
 <7954178+hanlz@users.noreply.github.com>
Date: Mon, 21 May 2018 17:40:56 +0800
Subject: [PATCH 0647/1225] Move the warning message output to standard error.

`ffmpeg -version` gives "ffmpeg version 2.8.14-0ubuntu0.16.04.1 Copyright (c) 2000-2018 the FFmpeg developers" on Ubuntu Xenial, which make int() failed during extracting version code.
---
 src/you_get/processor/ffmpeg.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 89d53e5009..1e3bd7eb0b 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -3,6 +3,7 @@
 import logging
 import os.path
 import subprocess
+import sys
 from ..util.strings import parameterize
 from ..common import print_more_compatible as print
 
@@ -25,8 +26,8 @@ def get_usable_ffmpeg(cmd):
         try:
             version = [int(i) for i in vers[2].split('.')]
         except:
-            print('It seems that your ffmpeg is a nightly build.')
-            print('Please switch to the latest stable if merging failed.')
+            print('It seems that your ffmpeg is a nightly build.', file=sys.stderr)
+            print('Please switch to the latest stable if merging failed.', file=sys.stderr)
             version = [1, 0]
         return cmd, 'ffprobe', version
     except:

From d26482b9a92ccaaecd683dc4fb5f17a6519ce417 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 21 May 2018 22:59:51 +0200
Subject: [PATCH 0648/1225] [twitter] prevent mobile redirection

---
 src/you_get/extractors/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 894439aa45..9cc3c5c74c 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -19,7 +19,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         url = 'https://' + match1(url, r'//mobile\.(.+)')
 
     if re.match(r'https?://twitter\.com/i/moments/', url): # moments
-        html = get_html(url)
+        html = get_html(url, faker=True)
         paths = re.findall(r'data-permalink-path="([^"]+)"', html)
         for path in paths:
             twitter_download('https://twitter.com' + path,
@@ -29,7 +29,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    html = get_html(url)
+    html = get_html(url, faker=True)
     screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
     item_id = r1(r'data-item-id="([^"]*)"', html) or \

From 8c0cae47fcf8c6edff8b08e9bd4d2ccce709e46e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 May 2018 19:16:31 +0200
Subject: [PATCH 0649/1225] [instagram] download video_url

---
 src/you_get/extractors/instagram.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index e06eba00c2..332d9b6155 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -27,6 +27,8 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             for edge in edges:
                 title = edge['node']['shortcode']
                 image_url = edge['node']['display_url']
+                if 'video_url' in edge['node']:
+                    image_url = edge['node']['video_url']
                 ext = image_url.split('.')[-1]
                 size = int(get_head(image_url)['Content-Length'])
                 print_info(site_info, title, ext, size)
@@ -39,6 +41,8 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         else:
             title = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['shortcode']
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
+            if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
+                image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
             ext = image_url.split('.')[-1]
             size = int(get_head(image_url)['Content-Length'])
             print_info(site_info, title, ext, size)

From b7603a4db0cbe1578f471da946cfa07c2de01736 Mon Sep 17 00:00:00 2001
From: "Agent Fitz ;-)" <fitz.bao@outlook.com>
Date: Tue, 29 May 2018 21:36:29 +0800
Subject: [PATCH 0650/1225] =?UTF-8?q?=E8=85=BE=E8=AE=AF=E8=A7=86=E9=A2=91?=
 =?UTF-8?q?=E9=BB=98=E8=AE=A4=E4=B8=8B=E8=BD=BDSHD=E8=A7=86=E9=A2=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

现在可以直接下载720P(SHD)的腾讯视频
---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 15116b0c73..4a67c57c44 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -9,7 +9,7 @@
 from urllib.parse import urlparse,parse_qs
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform=11&defnpayver=1&vid={}'.format(vid)
+    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform=11&defnpayver=1&defn=shd&vid={}'.format(vid)
     info = get_content(info_api)
     video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
 

From 3653420fe9965df30e63e7ed0495d58fa4538195 Mon Sep 17 00:00:00 2001
From: kiss4u <hljslmy@163.com>
Date: Sat, 2 Jun 2018 23:15:44 +0800
Subject: [PATCH 0651/1225] fix some url format from v.qq.com

https://v.qq.com/x/page/w0674l9yrrh.html
http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr
---
 src/you_get/extractors/acfun.py    |  2 +-
 src/you_get/extractors/bilibili.py |  2 +-
 src/you_get/extractors/qq.py       | 29 +++++++++++++++++++++++------
 3 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index c521422f60..4b45c5e962 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -65,7 +65,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     elif sourceType == 'tudou':
         tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'qq':
-        qq_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
+        qq_download_by_vid(sourceId, title, True, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'letv':
         letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
     elif sourceType == 'zhuzhan':
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 916782af92..9e2b8bc068 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -169,7 +169,7 @@ def entry(self, **kwargs):
             tc_flashvars = tc_flashvars.group(1)
         if tc_flashvars is not None:
             self.out = True
-            qq_download_by_vid(tc_flashvars, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
+            qq_download_by_vid(tc_flashvars, self.title, True, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
             return
 
         has_plist = re.search(r'"page":2', self.page)
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 15116b0c73..60fb751f3a 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -8,8 +8,14 @@
 from .qie_video import download_by_url as qie_video_download
 from urllib.parse import urlparse,parse_qs
 
-def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
-    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform=11&defnpayver=1&vid={}'.format(vid)
+def qq_download_by_vid(vid, title, default_from, output_dir='.', merge=True, info_only=False):
+
+    if default_from:
+        platform = 11
+    else:
+        platform = 4100201
+
+    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform={}&defnpayver=1&vid={}'.format(platform, vid)
     info = get_content(info_api)
     video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
 
@@ -17,7 +23,8 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     title = video_json['vl']['vi'][0]['ti']
     host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
     streams = video_json['fl']['fi']
-    seg_cnt = video_json['vl']['vi'][0]['cl']['fc']
+    seg_cnt = fc_cnt = video_json['vl']['vi'][0]['cl']['fc']
+
     filename = video_json['vl']['vi'][0]['fn']
     if seg_cnt == 0:
         seg_cnt = 1
@@ -39,7 +46,10 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
         # fix some error cases("check vid&filename failed" and "format invalid")
         # https://v.qq.com/x/page/q06058th9ll.html
         # https://v.qq.com/x/page/t060789a21e.html
-        if seg_cnt == 1:
+
+        if fc_cnt == 0:
+            # fix jason error 
+            # https://v.qq.com/x/page/w0674l9yrrh.html
             part_format_id = video_json['vl']['vi'][0]['cl']['keyid'].split('.')[-1]
         else:
             part_format_id = video_json['vl']['vi'][0]['cl']['ci'][part - 1]['keyid'].split('.')[1]
@@ -112,6 +122,8 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
+    default_from = True
+
     if re.match(r'https?://egame.qq.com/live\?anchorid=(\d+)', url):
         from . import qq_egame
         qq_egame.qq_egame_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
@@ -134,7 +146,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         content = get_content(url)
         vids = matchall(content, [r'\?vid=(\w+)'])
         for vid in vids:
-            qq_download_by_vid(vid, vid, output_dir, merge, info_only)
+            qq_download_by_vid(vid, vid, default_from, output_dir, merge, info_only)
         return
 
     if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
@@ -165,7 +177,12 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = match1(content, r'"title":"([^"]+)"') if not title else title
         title = vid if not title else title #general fallback
 
-    qq_download_by_vid(vid, title, output_dir, merge, info_only)
+        if 'v.sports.qq.com' in url:
+            # fix url forbidden
+            # http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr
+            default_from = False
+            
+    qq_download_by_vid(vid, title, default_from, output_dir, merge, info_only)
 
 site_info = "QQ.com"
 download = qq_download

From 928c8ccbe356800e582c8f0d60901da555ef7631 Mon Sep 17 00:00:00 2001
From: kiss4u <hljslmy@163.com>
Date: Sun, 3 Jun 2018 09:10:46 +0800
Subject: [PATCH 0652/1225] modify comments

---
 src/you_get/extractors/qq.py | 18 ++++--------------
 1 file changed, 4 insertions(+), 14 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 60fb751f3a..c3c653a8ff 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -13,6 +13,7 @@ def qq_download_by_vid(vid, title, default_from, output_dir='.', merge=True, inf
     if default_from:
         platform = 11
     else:
+        # fix return {,"msg":"cannot play outside"}
         platform = 4100201
 
     info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform={}&defnpayver=1&vid={}'.format(platform, vid)
@@ -32,24 +33,13 @@ def qq_download_by_vid(vid, title, default_from, output_dir='.', merge=True, inf
         fn_pre, magic_str, video_type = filename.split('.')
 
     best_quality = streams[-1]['name']
-    #part_format_id = streams[-1]['id']
 
     part_urls= []
     total_size = 0
     for part in range(1, seg_cnt+1):
-        #if seg_cnt == 1 and video_json['vl']['vi'][0]['vh'] <= 480:
-        #    filename = fn_pre + '.mp4'
-        #else:
-        #    filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
-        #filename = fn_pre + '.p' + str(part_format_id % 10000) + '.' + str(part) + '.mp4'
-
-        # fix some error cases("check vid&filename failed" and "format invalid")
-        # https://v.qq.com/x/page/q06058th9ll.html
-        # https://v.qq.com/x/page/t060789a21e.html
-
         if fc_cnt == 0:
-            # fix jason error 
-            # https://v.qq.com/x/page/w0674l9yrrh.html
+            # fix json parsing error
+            # example:https://v.qq.com/x/page/w0674l9yrrh.html
             part_format_id = video_json['vl']['vi'][0]['cl']['keyid'].split('.')[-1]
         else:
             part_format_id = video_json['vl']['vi'][0]['cl']['ci'][part - 1]['keyid'].split('.')[1]
@@ -179,7 +169,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         if 'v.sports.qq.com' in url:
             # fix url forbidden
-            # http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr
+            # example:http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr
             default_from = False
             
     qq_download_by_vid(vid, title, default_from, output_dir, merge, info_only)

From a10c98c5ab9aa5dc37e0c1baa652a9f456b2ca25 Mon Sep 17 00:00:00 2001
From: kiss4u <hljslmy@163.com>
Date: Mon, 4 Jun 2018 00:43:15 +0800
Subject: [PATCH 0653/1225] support for zhibo.tv

<http://v.zhibo.tv/>
example: http://v.zhibo.tv/31609372
<http://video.zhibo.tv/>
example: http://video.zhibo.tv/video/details/d103057f-663e-11e8-9d83-525400ccac43.html
---
 README.md                          |  1 +
 src/you_get/common.py              |  7 ++--
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/zhibo.py    | 52 ++++++++++++++++++++++++++++++
 4 files changed, 58 insertions(+), 3 deletions(-)
 create mode 100644 src/you_get/extractors/zhibo.py

diff --git a/README.md b/README.md
index 86c5e4e958..f6f8efdcac 100644
--- a/README.md
+++ b/README.md
@@ -416,6 +416,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
+| 中国体育(TV) | <http://v.zhibo.tv/> </br><http://video.zhibo.tv/>    |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index e30008549c..30e533f1a4 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -24,6 +24,7 @@
 SITES = {
     '163'              : 'netease',
     '56'               : 'w56',
+    '365yg'            : 'toutiao',
     'acfun'            : 'acfun',
     'archive'          : 'archive',
     'baidu'            : 'baidu',
@@ -64,6 +65,7 @@
     'iqiyi'            : 'iqiyi',
     'ixigua'           : 'ixigua',
     'isuntv'           : 'suntv',
+    'iwara'            : 'iwara',
     'joy'              : 'joy',
     'kankanews'        : 'bilibili',
     'khanacademy'      : 'khan',
@@ -82,6 +84,7 @@
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
     'musicplayon'      : 'musicplayon',
+    'miaopai'          : 'yixia',
     'naver'            : 'naver',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
@@ -118,14 +121,12 @@
     'xiaojiadianvideo' : 'fc2video',
     'ximalaya'         : 'ximalaya',
     'yinyuetai'        : 'yinyuetai',
-    'miaopai'          : 'yixia',
     'yizhibo'          : 'yizhibo',
     'youku'            : 'youku',
-    'iwara'            : 'iwara',
     'youtu'            : 'youtube',
     'youtube'          : 'youtube',
     'zhanqi'           : 'zhanqi',
-    '365yg'            : 'toutiao',
+    'zhibo'            : 'zhibo',
 }
 
 dry_run = False
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index ec9e86ae42..649a911f36 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -88,3 +88,4 @@
 from .khan import *
 from .zhanqi import *
 from .kuaishou import *
+from .zhibo import *
\ No newline at end of file
diff --git a/src/you_get/extractors/zhibo.py b/src/you_get/extractors/zhibo.py
new file mode 100644
index 0000000000..4aaa293e7d
--- /dev/null
+++ b/src/you_get/extractors/zhibo.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+
+__all__ = ['zhibo_download']
+
+from ..common import *
+
+def zhibo_vedio_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    # http://video.zhibo.tv/video/details/d103057f-663e-11e8-9d83-525400ccac43.html
+
+    html = get_html(url)
+    title = r1(r'<title>([\s\S]*)</title>', html)
+    total_size = 0
+    part_urls= []
+
+    video_html = r1(r'<script type="text/javascript">([\s\S]*)</script></head>', html)
+
+    # video_guessulike = r1(r"window.xgData =([s\S'\s\.]*)\'\;[\s\S]*window.vouchData", video_html) 
+    video_url = r1(r"window.vurl = \'([s\S'\s\.]*)\'\;[\s\S]*window.imgurl", video_html)
+    part_urls.append(video_url)
+    ext = video_url.split('.')[-1]
+
+    print_info(site_info, title, ext, total_size)
+    if not info_only:
+        download_urls(part_urls, title, ext, total_size, output_dir=output_dir, merge=merge)
+
+
+def zhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
+    if 'video.zhibo.tv' in url:
+        zhibo_vedio_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
+
+    # if 'v.zhibo.tv' in url:
+    # http://v.zhibo.tv/31609372
+    html = get_html(url)
+    title = r1(r'<title>([\s\S]*)</title>', html)
+    is_live = r1(r"window.videoIsLive=\'([s\S'\s\.]*)\'\;[\s\S]*window.resDomain", html)
+    if is_live is not "1":
+        raise ValueError("The live stream is not online! (Errno:%s)" % is_live)
+
+    ourStreamName = r1(r"window.ourStreamName=\'([s\S'\s\.]*)\'\;[\s\S]*window.rtmpDefaultSource", html)
+    rtmpPollUrl = r1(r"window.rtmpPollUrl=\'([s\S'\s\.]*)\'\;[\s\S]*window.hlsDefaultSource", html)
+
+    #real_url = 'rtmp://220.194.213.56/live.zhibo.tv/8live/' + ourStreamName
+    real_url = rtmpPollUrl + ourStreamName
+
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_url_ffmpeg(real_url, title, 'flv', params={}, output_dir=output_dir, merge=merge)
+
+site_info = "zhibo.tv"
+download = zhibo_download
+download_playlist = playlist_not_supported('zhibo')

From ebbe13e88e78e2f6eff80a495ad5a90580391d49 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Jun 2018 00:29:00 +0200
Subject: [PATCH 0654/1225] [universal] a URL with space is not a good URL

---
 src/you_get/extractors/universal.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 6a1c2d302f..57b9b2d10b 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -67,12 +67,12 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^;"\'\\]+' + i + r'[^;"\'\\]*)', page)
+            urls += re.findall(r'(https?://[^ ;"\'\\]+' + i + r'[^ ;"\'\\]*)', page)
 
             p_urls = re.findall(r'(https?%3A%2F%2F[^;&]+' + i + r'[^;&]*)', page)
             urls += [parse.unquote(url) for url in p_urls]
 
-            q_urls = re.findall(r'(https?:\\\\/\\\\/[^;"\']+' + i + r'[^;"\']*)', page)
+            q_urls = re.findall(r'(https?:\\\\/\\\\/[^ ;"\']+' + i + r'[^ ;"\']*)', page)
             urls += [url.replace('\\\\/', '/') for url in q_urls]
 
         # a link href to an image is often an interesting one

From 52e6a7482d99bc5d928f81e27309ff964c17c7a7 Mon Sep 17 00:00:00 2001
From: Justlearnm0re <driverCzn@users.noreply.github.com>
Date: Sun, 24 Jun 2018 17:41:58 +0800
Subject: [PATCH 0655/1225] fix cid match

The old regex is broken, fix it with new one.
---
 src/you_get/extractors/bilibili.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9e2b8bc068..523abbdb79 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -177,7 +177,9 @@ def entry(self, **kwargs):
             log.w('This page contains a playlist. (use --playlist to download all videos.)')
 
         try:
-            cid = re.search(r'cid=(\d+)', self.page).group(1)
+            page_list = json.loads(re.search(r'"pages":(\[.*?\])', self.page).group(1))
+            index_id = int(re.search(r'index_(\d+)', self.url).group(1))
+            cid = page_list[index_id-1]['cid'] # change cid match rule
         except:
             cid = re.search(r'"cid":(\d+)', self.page).group(1)
         if cid is not None:

From 1adb799c869c50ae19c643db9091912dbb68bac1 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Mon, 25 Jun 2018 16:55:13 +0800
Subject: [PATCH 0656/1225] Update douyutv.py

douyu update
---
 src/you_get/extractors/douyutv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index 72a41a0ad4..bdcea45827 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -45,9 +45,9 @@ def douyutv_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         douyutv_video_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return
 
-    url = re.sub(r'[w.]*douyu.com', 'm.douyu.com', url)
+    url = re.sub(r'.*douyu.com','https://m.douyu.com/room', url)
     html = get_content(url, headers)
-    room_id_patt = r'room_id\s*:\s*(\d+),'
+    room_id_patt = r'"rid"\s*:\s*(\d+),'
     room_id = match1(html, room_id_patt)
     if room_id == "0":
         room_id = url[url.rfind('/') + 1:]

From 4f00ca5b8da7dcaf51a3d5f91e168c53f7efd156 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 27 Jun 2018 22:18:27 +0200
Subject: [PATCH 0657/1225] [bilibili] warn when target URL is a playlist and
 --playlist is not used

---
 src/you_get/extractors/bilibili.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 523abbdb79..cd71b071f8 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -173,7 +173,7 @@ def entry(self, **kwargs):
             return
 
         has_plist = re.search(r'"page":2', self.page)
-        if has_plist:
+        if has_plist and not kwargs.get('playlist'):
             log.w('This page contains a playlist. (use --playlist to download all videos.)')
 
         try:
@@ -341,6 +341,7 @@ def parse_cid_playurl(xml):
 
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url])[0]
+    kwargs['playlist'] = True
     # a bangumi here? possible?
     if 'live.bilibili' in url:
         site.download_by_url(url)

From 93c08277d20b373d270b488bf1e11d20dda7e17e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 27 Jun 2018 22:50:10 +0200
Subject: [PATCH 0658/1225] [bilibili] better subtitle with page no

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index cd71b071f8..9374959601 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -360,7 +360,7 @@ def bilibili_download_playlist_by_url(url, **kwargs):
         page_cnt = len(page_list)
         for no in range(1, page_cnt+1):
             page_url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, no)
-            subtitle = page_list[no-1]['pagename']
+            subtitle = '#%s. %s'% (page_list[no-1]['page'], page_list[no-1]['pagename'])
             Bilibili().download_by_url(page_url, subtitle=subtitle, **kwargs)
 
 site = Bilibili()

From 1171be87c7dc2d2ae66c1c37d02ae9723fe414e5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 27 Jun 2018 23:21:42 +0200
Subject: [PATCH 0659/1225] [baidu] squanch this

---
 src/you_get/extractors/baidu.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 6f558e31dc..d0146217e7 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -129,6 +129,15 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
+            vhsrc = re.findall(r'vhsrc="([^"]+)"', html)
+            if vhsrc is not None:
+                ext = 'mp4'
+                size = url_size(vhsrc[0])
+                print_info(site_info, title, ext, size)
+                if not info_only:
+                    download_urls(vhsrc, title, ext, size,
+                                  output_dir=output_dir, merge=False)
+
             items = re.findall(
                 r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
             urls = ['http://imgsrc.baidu.com/forum/pic/item/' + i

From 18af8f36e2e9a12761ae582527818b8af3c3e891 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 27 Jun 2018 23:35:15 +0200
Subject: [PATCH 0660/1225] version 0.4.1099

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 64ef890f8b..9f3287d936 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1077'
+__version__ = '0.4.1099'

From 4a3f1b5bd2c87d43667730578df12ac121d11322 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 29 Jun 2018 15:04:42 +0200
Subject: [PATCH 0661/1225] [miaopai] squanch that

---
 src/you_get/extractors/yixia.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index ccaaf5462b..ff45730d66 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -51,10 +51,10 @@ def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         yixia_download_by_scid = yixia_miaopai_download_by_scid
         site_info = "Yixia Miaopai"
 
-        scid = match1(url, r'miaopai\.com/show/channel/(.+)\.htm') or \
-               match1(url, r'miaopai\.com/show/(.+)\.htm') or \
-               match1(url, r'm\.miaopai\.com/show/channel/(.+)\.htm') or \
-               match1(url, r'm\.miaopai\.com/show/channel/(.+)')
+        scid = match1(url, r'miaopai\.com/show/channel/([^.]+)\.htm') or \
+               match1(url, r'miaopai\.com/show/([^.]+)\.htm') or \
+               match1(url, r'm\.miaopai\.com/show/channel/([^.]+)\.htm') or \
+               match1(url, r'm\.miaopai\.com/show/channel/([^.]+)')
 
     elif 'xiaokaxiu.com' in hostname:  #Xiaokaxiu
         yixia_download_by_scid = yixia_xiaokaxiu_download_by_scid

From 503ff846f77225ee373f58c016da6428d8d5a2b7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Jul 2018 13:23:48 +0200
Subject: [PATCH 0662/1225] [common] do not coerce headers into a dict

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 30e533f1a4..73192e6126 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -505,7 +505,7 @@ def get_head(url, headers={}, get_method='HEAD'):
         req = request.Request(url)
     req.get_method = lambda: get_method
     res = urlopen_with_retry(req)
-    return dict(res.headers)
+    return res.headers
 
 
 def url_info(url, faker=False, headers={}):

From 37e2a798972d2ad7d73abfc86ab7c54a2f42a077 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Jul 2018 13:48:22 +0200
Subject: [PATCH 0663/1225] [common] wubba lubba dub dub

---
 src/you_get/common.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 73192e6126..f4c37f7188 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1594,15 +1594,8 @@ def url_to_module(url):
             url
         )
     else:
-        import http.client
-        video_host = r1(r'https?://([^/]+)/', url)  # .cn could be removed
-        if url.startswith('https://'):
-            conn = http.client.HTTPSConnection(video_host)
-        else:
-            conn = http.client.HTTPConnection(video_host)
-        conn.request('HEAD', video_url, headers=fake_headers)
-        res = conn.getresponse()
-        location = res.getheader('location')
+        location = get_location(url)
+
         if location and location != url and not location.startswith('/'):
             return url_to_module(location)
         else:

From 3e8927959836b96982a06fe922a0946cfca52ed2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Jul 2018 15:47:54 +0200
Subject: [PATCH 0664/1225] [common] make get_location accept headers

---
 src/you_get/common.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f4c37f7188..c8b0f80ba4 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -369,13 +369,16 @@ def get_decoded_html(url, faker=False):
         return data
 
 
-def get_location(url):
+def get_location(url, headers=None, get_method='HEAD'):
     logging.debug('get_location: %s' % url)
 
-    response = request.urlopen(url)
-    # urllib will follow redirections and it's too much code to tell urllib
-    # not to do that
-    return response.geturl()
+    if headers:
+        req = request.Request(url, headers=headers)
+    else:
+        req = request.Request(url)
+    req.get_method = lambda: get_method
+    res = urlopen_with_retry(req)
+    return res.geturl()
 
 
 def urlopen_with_retry(*args, **kwargs):
@@ -1594,7 +1597,10 @@ def url_to_module(url):
             url
         )
     else:
-        location = get_location(url)
+        try:
+            location = get_location(url) # t.co isn't happy with fake_headers
+        except:
+            location = get_location(url, headers=fake_headers)
 
         if location and location != url and not location.startswith('/'):
             return url_to_module(location)

From d503237508167e761f835045d36cdec0a928d31f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Jul 2018 15:48:22 +0200
Subject: [PATCH 0665/1225] [common] dumb

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c8b0f80ba4..97bc93a559 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -499,7 +499,7 @@ def urls_size(urls, faker=False, headers={}):
     return sum([url_size(url, faker=faker, headers=headers) for url in urls])
 
 
-def get_head(url, headers={}, get_method='HEAD'):
+def get_head(url, headers=None, get_method='HEAD'):
     logging.debug('get_head: %s' % url)
 
     if headers:

From 9ab4bfbf3e4ef99bd67c2bcba4eaa4398edc1cbd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 1 Jul 2018 15:50:08 +0200
Subject: [PATCH 0666/1225] [common] update UA

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 97bc93a559..b19d602f14 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -143,7 +143,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:51.0) Gecko/20100101 Firefox/51.0',  # noqa
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0',  # noqa
 }
 
 if sys.stdout.isatty():

From e635628639cc6f709dbc84dd8e45c3bdcef7758a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 3 Jul 2018 14:49:51 +0200
Subject: [PATCH 0667/1225] [instagram] no more ?

---
 src/you_get/extractors/instagram.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 332d9b6155..65fc01f510 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -29,6 +29,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 image_url = edge['node']['display_url']
                 if 'video_url' in edge['node']:
                     image_url = edge['node']['video_url']
+                image_url = image_url.split('?')[0]
                 ext = image_url.split('.')[-1]
                 size = int(get_head(image_url)['Content-Length'])
                 print_info(site_info, title, ext, size)
@@ -43,6 +44,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
             if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
                 image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
+            image_url = image_url.split('?')[0]
             ext = image_url.split('.')[-1]
             size = int(get_head(image_url)['Content-Length'])
             print_info(site_info, title, ext, size)

From 196e94bdfff870e795b6aab5618c33166378fe4a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 12 Jul 2018 01:03:01 +0200
Subject: [PATCH 0668/1225] [baidu] because the fleeb has all of the fleeb
 juice

---
 src/you_get/extractors/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index d0146217e7..c9d64547f1 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -129,7 +129,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            vhsrc = re.findall(r'vhsrc="([^"]+)"', html)
+            vhsrc = re.findall(r'"BDE_Image" src="([^"]+)"', html)
             if vhsrc is not None:
                 ext = 'mp4'
                 size = url_size(vhsrc[0])

From a07ba1a5dfc8f0a46b957426a1dc528a90c589b8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 14 Jul 2018 14:54:21 +0200
Subject: [PATCH 0669/1225] [baidu] it's important that the fleeb is rubbed

---
 src/you_get/extractors/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index c9d64547f1..1392e7de9c 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -129,7 +129,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            vhsrc = re.findall(r'"BDE_Image" src="([^"]+)"', html)
+            vhsrc = re.findall(r'"BDE_Image" src="([^"]+)"', html) or re.findall(r'vhsrc="([^"]+)"', html)
             if vhsrc is not None:
                 ext = 'mp4'
                 size = url_size(vhsrc[0])

From 50216593e439b6e940e868a9f98c4475ee3636f5 Mon Sep 17 00:00:00 2001
From: Fangzhou Li <riophaelee@gmail.com>
Date: Mon, 16 Jul 2018 04:22:13 +0800
Subject: [PATCH 0670/1225] [util] improve compatibility with WSL

---
 src/you_get/util/fs.py |  8 ++++----
 src/you_get/util/os.py | 30 ++++++++++++++++++++++++++++++
 tests/test_util.py     |  7 ++++---
 3 files changed, 38 insertions(+), 7 deletions(-)
 create mode 100644 src/you_get/util/os.py

diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
index d49a117d62..b6b7069afa 100644
--- a/src/you_get/util/fs.py
+++ b/src/you_get/util/fs.py
@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
-import platform
+from .os import detect_os
 
-def legitimize(text, os=platform.system()):
+def legitimize(text, os=detect_os()):
     """Converts a string to a valid filename.
     """
 
@@ -13,7 +13,7 @@ def legitimize(text, os=platform.system()):
         ord('|'): '-',
     })
 
-    if os == 'Windows':
+    if os == 'windows' or os == 'cygwin' or os == 'wsl':
         # Windows (non-POSIX namespace)
         text = text.translate({
             # Reserved in Windows VFAT and NTFS
@@ -31,7 +31,7 @@ def legitimize(text, os=platform.system()):
         })
     else:
         # *nix
-        if os == 'Darwin':
+        if os == 'mac':
             # Mac OS HFS+
             text = text.translate({
                 ord(':'): '-',
diff --git a/src/you_get/util/os.py b/src/you_get/util/os.py
new file mode 100644
index 0000000000..11730e28c9
--- /dev/null
+++ b/src/you_get/util/os.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+from platform import system
+
+def detect_os():
+    """Detect operating system.
+    """
+
+    # Inspired by:
+    # https://github.com/scivision/pybashutils/blob/78b7f2b339cb03b1c37df94015098bbe462f8526/pybashutils/windows_linux_detect.py
+
+    syst = system().lower()
+    os = 'unknown'
+
+    if 'cygwin' in syst:
+        os = 'cygwin'
+    elif 'darwin' in syst:
+        os = 'mac'
+    elif 'linux' in syst:
+        os = 'linux'
+        # detect WSL https://github.com/Microsoft/BashOnWindows/issues/423
+        with open('/proc/version', 'r') as f:
+            if 'microsoft' in f.read().lower():
+                os = 'wsl'
+    elif 'windows' in syst:
+        os = 'windows'
+    elif 'bsd' in syst:
+        os = 'bsd'
+
+    return os
diff --git a/tests/test_util.py b/tests/test_util.py
index 239083bc6c..88743b0377 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -6,6 +6,7 @@
 
 class TestUtil(unittest.TestCase):
     def test_legitimize(self):
-        self.assertEqual(legitimize("1*2", os="Linux"), "1*2")
-        self.assertEqual(legitimize("1*2", os="Darwin"), "1*2")
-        self.assertEqual(legitimize("1*2", os="Windows"), "1-2")
+        self.assertEqual(legitimize("1*2", os="linux"), "1*2")
+        self.assertEqual(legitimize("1*2", os="mac"), "1*2")
+        self.assertEqual(legitimize("1*2", os="windows"), "1-2")
+        self.assertEqual(legitimize("1*2", os="wsl"), "1-2")

From ddf67aadb5f90ececd2246c7a6302a66b630eeac Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 17 Jul 2018 17:59:31 +0200
Subject: [PATCH 0671/1225] [baidu] squanch this, mofo

---
 src/you_get/extractors/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 1392e7de9c..b30c9d86ed 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -129,7 +129,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            vhsrc = re.findall(r'"BDE_Image" src="([^"]+)"', html) or re.findall(r'vhsrc="([^"]+)"', html)
+            vhsrc = re.findall(r'"BDE_Image"[^>]+src="([^"]+)"', html) or re.findall(r'vhsrc="([^"]+)"', html)
             if vhsrc is not None:
                 ext = 'mp4'
                 size = url_size(vhsrc[0])

From 80aa34f538f52f60484609405d07fc62377827bc Mon Sep 17 00:00:00 2001
From: ellipse42 <ellipse42@qq.com>
Date: Sat, 21 Jul 2018 06:26:19 +0800
Subject: [PATCH 0672/1225] [miaopai] support not fixed length fid

---
 src/you_get/extractors/miaopai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 6d9a79c719..f37d45b004 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -31,7 +31,7 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
 
 #----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    fid = match1(url, r'\?fid=(\d{4}:\w{32})')
+    fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)
     elif '/p/230444' in url:

From feffd883ea30a4b8f6af9cbd3e631489e051919c Mon Sep 17 00:00:00 2001
From: hellsof <hellsof@hotmail.com>
Date: Sat, 28 Jul 2018 13:51:43 +0800
Subject: [PATCH 0673/1225] support view.inews.qq.com/a/20180521V0Z9MH00
 https://kuaibao.qq.com/s/20180521V0Z9MH00
 https://v.qq.com/x/cover/t0fqsm1y83r8v5j/a0026nvw5jr.html

---
 src/you_get/extractors/qq.py | 51 +++++++++++++++++++-----------------
 1 file changed, 27 insertions(+), 24 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 7f2f4acc22..e39bf2e30d 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -2,28 +2,25 @@
 
 __all__ = ['qq_download']
 
-from ..common import *
-from ..util.log import *
 from .qie import download as qieDownload
 from .qie_video import download_by_url as qie_video_download
-from urllib.parse import urlparse,parse_qs
-
-def qq_download_by_vid(vid, title, default_from, output_dir='.', merge=True, info_only=False):
+from ..common import *
 
-    if default_from:
-        platform = 11
-    else:
-        # fix return {,"msg":"cannot play outside"}
-        platform = 4100201
 
-    info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform=11&defnpayver=1&defn=shd&vid={}'.format(vid)
-    info = get_content(info_api)
-    video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
+def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
 
+    # http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr https://v.qq.com/x/cover/t0fqsm1y83r8v5j/a0026nvw5jr.html
+    video_json = None
+    platforms = [4100201, 11]
+    for platform in platforms:
+        info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform={}&defnpayver=1&defn=shd&vid={}'.format(platform, vid)
+        info = get_content(info_api)
+        video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
+        if not video_json.get('msg')=='cannot play outside':
+            break
     fn_pre = video_json['vl']['vi'][0]['lnk']
     title = video_json['vl']['vi'][0]['ti']
     host = video_json['vl']['vi'][0]['ul']['ui'][0]['url']
-    streams = video_json['fl']['fi']
     seg_cnt = fc_cnt = video_json['vl']['vi'][0]['cl']['fc']
 
     filename = video_json['vl']['vi'][0]['fn']
@@ -32,8 +29,6 @@ def qq_download_by_vid(vid, title, default_from, output_dir='.', merge=True, inf
     else:
         fn_pre, magic_str, video_type = filename.split('.')
 
-    best_quality = streams[-1]['name']
-
     part_urls= []
     total_size = 0
     for part in range(1, seg_cnt+1):
@@ -112,7 +107,6 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
 
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
-    default_from = True
 
     if re.match(r'https?://egame.qq.com/live\?anchorid=(\d+)', url):
         from . import qq_egame
@@ -136,10 +130,18 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         content = get_content(url)
         vids = matchall(content, [r'\?vid=(\w+)'])
         for vid in vids:
-            qq_download_by_vid(vid, vid, default_from, output_dir, merge, info_only)
+            qq_download_by_vid(vid, vid, output_dir, merge, info_only)
         return
 
-    if 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
+    if 'kuaibao.qq.com/s/' in url:
+        # https://kuaibao.qq.com/s/20180521V0Z9MH00
+        nid = match1(url, r'/s/([^/&?#]+)')
+        content = get_content('https://kuaibao.qq.com/getVideoRelate?id=' + nid)
+        info_json = json.loads(content)
+        vid=info_json['videoinfo']['vid']
+        title=info_json['videoinfo']['title']
+    elif 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
+        # http://daxue.qq.com/content/content/id/2321
         content = get_content(url)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
@@ -148,6 +150,11 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = match1(url, r'\bvid=(\w+)')
         # for embedded URLs; don't know what the title is
         title = vid
+    elif 'view.inews.qq.com' in url:
+        # view.inews.qq.com/a/20180521V0Z9MH00
+        content = get_content(url)
+        vid = match1(content, r'"vid":"(\w+)"')
+        title = match1(content, r'"title":"(\w+)"')
     else:
         content = get_content(url)
         #vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
@@ -167,12 +174,8 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = match1(content, r'"title":"([^"]+)"') if not title else title
         title = vid if not title else title #general fallback
 
-        if 'v.sports.qq.com' in url:
-            # fix url forbidden
-            # example:http://v.sports.qq.com/#/cover/t0fqsm1y83r8v5j/a0026nvw5jr
-            default_from = False
 
-    qq_download_by_vid(vid, title, default_from, output_dir, merge, info_only)
+    qq_download_by_vid(vid, title, output_dir, merge, info_only)
 
 site_info = "QQ.com"
 download = qq_download

From 506c5b5bc12f3b3597354cb31212b305227462c5 Mon Sep 17 00:00:00 2001
From: ehds <grephale@gmail.com>
Date: Sun, 29 Jul 2018 14:03:10 +0800
Subject: [PATCH 0674/1225] upadate kugou api

---
 src/you_get/extractors/kugou.py | 63 ++++++++++++++++++++++++++-------
 1 file changed, 50 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/kugou.py b/src/you_get/extractors/kugou.py
index 925bdf1c34..a98daac6d5 100644
--- a/src/you_get/extractors/kugou.py
+++ b/src/you_get/extractors/kugou.py
@@ -20,32 +20,69 @@ def kugou_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
         print_info(site_info, title, songtype, size)
         if not info_only:
             download_urls([url], title, ext, size, output_dir, merge=merge)
+    elif url.lower().find("hash")!=-1:
+        return kugou_download_by_hash(url,output_dir,merge,info_only)
     else:
         #for the www.kugou.com/
         return kugou_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
         # raise NotImplementedError(url)       
 
-def kugou_download_by_hash(title,hash_val,output_dir = '.', merge = True, info_only = False):
+
+def kugou_download_by_hash(url,output_dir = '.', merge = True, info_only = False):
     #sample
-    #url_sample:http://www.kugou.com/yy/album/single/536957.html
-    #hash ->key  md5(hash+kgcloud")->key  decompile swf
-    #cmd 4 for mp3 cmd 3 for m4a
-    key=hashlib.new('md5',(hash_val+"kgcloud").encode("utf-8")).hexdigest()
-    html=get_html("http://trackercdn.kugou.com/i/?pid=6&key=%s&acceptMp3=1&cmd=4&hash=%s"%(key,hash_val))
-    j=loads(html)
-    url=j['url']
+    #url_sample:http://www.kugou.com/song/#hash=93F7D2FC6E95424739448218B591AEAF&album_id=9019462
+    hash_val = match1(url,'hash=(\w+)')
+    album_id = match1(url,'album_id=(\d+)')
+    html = get_html("http://www.kugou.com/yy/index.php?r=play/getdata&hash={}&album_id={}".format(hash_val,album_id))
+    j =loads(html)
+    url = j['data']['play_url']
+    title = j['data']['audio_name']
+    # some songs cann't play because of copyright protection
+    if(url == ''):
+        return
     songtype, ext, size = url_info(url)
     print_info(site_info, title, songtype, size)
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge=merge)
 
 def kugou_download_playlist(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    html=get_html(url)
-    pattern=re.compile('title="(.*?)".* data="(\w*)\|.*?"')
-    pairs=pattern.findall(html)
-    for title,hash_val in pairs:
-        kugou_download_by_hash(title,hash_val,output_dir,merge,info_only)
+    urls=[]
+    
+    #download music leaderboard
+    #sample: http://www.kugou.com/yy/html/rank.html
+    if url.lower().find('rank') !=-1:
+        html=get_html(url)
+        pattern = re.compile('<a href="(http://.*?)" data-active=')
+        res = pattern.findall(html)
+        for song in res:
+            res = get_html(song)
+            pattern_url = re.compile('"hash":"(\w+)".*"album_id":(\d)+')
+            hash_val,album_id= res = pattern_url.findall(res)[0]
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(hash_val,album_id))
+    
+    # download album
+    # album sample:   http://www.kugou.com/yy/album/single/1645030.html
+    elif url.lower().find('album')!=-1:
+        html = get_html(url)
+        pattern = re.compile('var data=(\[.*?\]);')
+        res = pattern.findall(html)[0]
+        for v in json.loads(res):
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v['hash'],v['album_id']))
+
+    # download the playlist        
+    # playlist sample:http://www.kugou.com/yy/special/single/487279.html
+    else:
+        html = get_html(url)
+        pattern = re.compile('data="(\w+)\|(\d+)"')
+        for v in pattern.findall(html):
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v[0],v[1]))
+            print('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v[0],v[1]))
+
+    #download the list by hash
+    for url in urls:
+        kugou_download_by_hash(url,output_dir,merge,info_only)
 
+                
 
 site_info = "kugou.com"
 download = kugou_download

From 493e26149ce26c4444d6029547ee4aa0853b1a4f Mon Sep 17 00:00:00 2001
From: Gong Qijian <gongqijian@gmail.com>
Date: Sat, 4 Aug 2018 10:15:59 +0800
Subject: [PATCH 0675/1225] [bilibili] add subtitle for downloading single
 episode

---
 src/you_get/extractors/bilibili.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9374959601..103c5c9599 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -137,9 +137,18 @@ def prepare(self, **kwargs):
             m = re.search(r'property="og:title" content="([^"]+)"', self.page)
             if m is not None:
                 self.title = m.group(1)
+
         if 'subtitle' in kwargs:
             subtitle = kwargs['subtitle']
             self.title = '{} {}'.format(self.title, subtitle)
+        else:
+            m_pages = re.search(r'"pages":(\[[^\]]+])', self.page)
+            if m_pages is not None:
+                pages = json.loads(m_pages.group(1))
+                if len(pages) > 1:
+                    qs = dict(parse.parse_qsl(urllib.parse.urlparse(self.url).query))
+                    page = pages[int(qs.get('p', 1)) - 1]
+                    self.title = '{} #{}. {}'.format(self.title, page['page'], page['part'])
 
         if 'bangumi.bilibili.com/movie' in self.url:
             self.movie_entry(**kwargs)

From da8c982608c9308765e0960e08fc28cccb74b215 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Aug 2018 16:21:49 +0200
Subject: [PATCH 0676/1225] [twitter] fix #2609

---
 src/you_get/extractors/twitter.py | 50 +++++++++++--------------------
 1 file changed, 17 insertions(+), 33 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 9cc3c5c74c..8ed400dbb2 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -64,42 +64,26 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                               output_dir=output_dir)
 
     except: # extract video
-        # always use i/cards or videos url
-        if not re.match(r'https?://twitter.com/i/', url):
-            url = r1(r'<meta\s*property="og:video:url"\s*content="([^"]+)"', html)
-            if not url:
-                url = 'https://twitter.com/i/videos/%s' % item_id
-            try:
-                html = get_content(url)
-            except:
-                return
+        #i_url = 'https://twitter.com/i/videos/' + item_id
+        #i_content = get_content(i_url)
+        #js_url = r1(r'src="([^"]+)"', i_content)
+        #js_content = get_content(js_url)
+        #authorization = r1(r'"(Bearer [^"]+)"', js_content)
+        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
 
-        data_config = r1(r'data-config="([^"]*)"', html) or \
-            r1(r'data-player-config="([^"]*)"', html)
-        i = json.loads(unescape_html(data_config))
-        if 'video_url' in i:
-            source = i['video_url']
-            item_id = i['tweet_id']
-            page_title = "{} [{}]".format(screen_name, item_id)
-        elif 'playlist' in i:
-            source = i['playlist'][0]['source']
-            if not item_id: page_title = i['playlist'][0]['contentId']
-        elif 'vmap_url' in i:
-            vmap_url = i['vmap_url']
-            vmap = get_content(vmap_url)
-            source = r1(r'<MediaFile>\s*<!\[CDATA\[(.*)\]\]>', vmap)
-            item_id = i['tweet_id']
-            page_title = "{} [{}]".format(screen_name, item_id)
-        elif 'scribe_playlist_url' in i:
-            scribe_playlist_url = i['scribe_playlist_url']
-            return vine_download(scribe_playlist_url, output_dir, merge=merge, info_only=info_only)
+        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
+        ga_content = post_content(ga_url, headers={'authorization': authorization})
+        guest_token = json.loads(ga_content)['guest_token']
 
-        try:
-            urls = extract_m3u(source)
-        except:
-            urls = [source]
+        api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
+        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
+
+        info = json.loads(api_content)
+        variants = info['globalObjects']['tweets'][item_id]['extended_entities']['media'][0]['video_info']['variants']
+        variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
+        urls = [ variants[-1]['url'] ]
         size = urls_size(urls)
-        mime, ext = 'video/mp4', 'mp4'
+        mime, ext = variants[-1]['content_type'], 'mp4'
 
         print_info(site_info, page_title, mime, size)
         if not info_only:

From 711b3621d0a03069ddd8332177685a0ed3bcc9fd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Aug 2018 16:26:22 +0200
Subject: [PATCH 0677/1225] version 0.4.1118

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 9f3287d936..93d636e844 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1099'
+__version__ = '0.4.1118'

From 0ce55a2cf6f1feba51768c57887d952f53580005 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Aug 2018 17:06:41 +0200
Subject: [PATCH 0678/1225] [bilibili] nah (#2627)

---
 src/you_get/extractors/bilibili.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 103c5c9599..f3115ed224 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -142,9 +142,9 @@ def prepare(self, **kwargs):
             subtitle = kwargs['subtitle']
             self.title = '{} {}'.format(self.title, subtitle)
         else:
-            m_pages = re.search(r'"pages":(\[[^\]]+])', self.page)
-            if m_pages is not None:
-                pages = json.loads(m_pages.group(1))
+            playinfo = re.search(r'__INITIAL_STATE__=(.*?);\(function\(\)', self.page)
+            if playinfo is not None:
+                pages = json.loads(playinfo.group(1))['videoData']['pages']
                 if len(pages) > 1:
                     qs = dict(parse.parse_qsl(urllib.parse.urlparse(self.url).query))
                     page = pages[int(qs.get('p', 1)) - 1]

From c6d71ddf102af848741c3f5ca97fa46ff07d8806 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Aug 2018 17:12:11 +0200
Subject: [PATCH 0679/1225] version 0.4.1120

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 93d636e844..93a759dec2 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1118'
+__version__ = '0.4.1120'

From 3a41c3d22c52aa5a08b63fdc1335d6b1d1b0b4fa Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 10 Aug 2018 03:05:14 +0200
Subject: [PATCH 0680/1225] [processor.ffmpeg] shut the f up

---
 src/you_get/processor/ffmpeg.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 1e3bd7eb0b..a18188dac9 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -22,12 +22,10 @@ def get_usable_ffmpeg(cmd):
         out, err = p.communicate()
         vers = str(out, 'utf-8').split('\n')[0].split()
         assert (vers[0] == 'ffmpeg' and vers[2][0] > '0') or (vers[0] == 'avconv')
-        #set version to 1.0 for nightly build and print warning
         try:
-            version = [int(i) for i in vers[2].split('.')]
+            v = vers[2][1:] if vers[2][0] == 'n' else vers[2]
+            version = [int(i) for i in v.split('.')]
         except:
-            print('It seems that your ffmpeg is a nightly build.', file=sys.stderr)
-            print('Please switch to the latest stable if merging failed.', file=sys.stderr)
             version = [1, 0]
         return cmd, 'ffprobe', version
     except:

From b554f9f9bcf8f689a2ddf717361a0676f4cbd039 Mon Sep 17 00:00:00 2001
From: "edward.gao" <edward.gao@logicmonitor.com>
Date: Fri, 10 Aug 2018 17:26:29 +0800
Subject: [PATCH 0681/1225] Feature: support download bilibili space favorite
 folder files

---
 src/you_get/extractors/bilibili.py | 47 +++++++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f3115ed224..e0ebbac814 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -32,13 +32,13 @@ class Bilibili(VideoExtractor):
     SEC1 = '94aba54af9065f71de72f5508f1cd42e'
     SEC2 = '9b288147e5474dd2aa67085f716c560d'
     stream_types = [
-            {'id': 'hdflv'},
-            {'id': 'flv720'},
-            {'id': 'flv'},
-            {'id': 'hdmp4'},
-            {'id': 'mp4'},
-            {'id': 'live'},
-            {'id': 'vc'}
+        {'id': 'hdflv'},
+        {'id': 'flv720'},
+        {'id': 'flv'},
+        {'id': 'hdmp4'},
+        {'id': 'mp4'},
+        {'id': 'live'},
+        {'id': 'vc'}
     ]
     fmt2qlt = dict(hdflv=4, flv=3, hdmp4=2, mp4=1)
 
@@ -348,6 +348,36 @@ def parse_cid_playurl(xml):
         log.w(e)
         return [], 0
 
+def download_video_from_favlist(url, **kwargs):
+    # the url has format: https://space.bilibili.com/64169458/#/favlist?fid=1840028
+
+    m = re.search(r'space\.bilibili\.com/(\d+)/.*?fid=(\d+).*?', url)
+    vmid = ""
+    favid = ""
+    if m is not None:
+        vmid = m.group(1)
+        favid = m.group(2)
+        jsonresult = json.loads(get_content("https://api.bilibili.com/x/space/fav/arc?vmid={}&ps=300&fid={}&order=fav_time&tid=0&keyword=&pn=1&jsonp=jsonp".format(vmid, favid)))
+        print(jsonresult)
+        # log.wtf("Got files list for vmid" + vmid + " favid:" + favid)
+        if jsonresult['code'] != 0:
+            log.wtf("Fail to get the files of page " + jsonresult)
+            sys.exit(2)
+
+        else:
+            videos = jsonresult['data']['archives']
+            videocount = len(videos)
+            for i in range(videocount):
+                videoid = videos[i]["aid"]
+                videotitle = videos[i]["title"]
+                videourl = "https://www.bilibili.com/video/av{}".format(videoid)
+                print("Start downloading ", videotitle, " video ", videotitle)
+                Bilibili().download_by_url(videourl, subtitle=videotitle, **kwargs)
+
+    else:
+        log.wtf("Fail to parse the fav title" + url, "")
+
+
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url])[0]
     kwargs['playlist'] = True
@@ -363,6 +393,9 @@ def bilibili_download_playlist_by_url(url, **kwargs):
         for ep_id in ep_ids:
             ep_url = '#'.join([base_url, ep_id])
             Bilibili().download_by_url(ep_url, **kwargs)
+    elif 'favlist' in url:
+        # this a fav list folder
+        download_video_from_favlist(url, **kwargs)
     else:
         aid = re.search(r'av(\d+)', url).group(1)
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))

From 611cb6acdffac4a1760173eecb14e547a0ce586e Mon Sep 17 00:00:00 2001
From: "edward.gao" <edward.gao@logicmonitor.com>
Date: Fri, 10 Aug 2018 18:15:10 +0800
Subject: [PATCH 0682/1225] The bilibili got 502 occasionally

---
 tests/test.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 6562d7cad4..4a402f1ed7 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import unittest
+import urllib
 
 from you_get.extractors import (
     imgur,
@@ -32,12 +33,23 @@ def test_youtube(self):
         )
 
     def test_bilibili(self):
-        bilibili.download(
-            'https://www.bilibili.com/video/av16907446/', info_only=True
-        )
-        bilibili.download(
-            'https://www.bilibili.com/video/av13228063/', info_only=True
-        )
+        maxRetry = 3
+
+        # the bilibi is not stable, it got 502 occasionally
+        for i in range(maxRetry):
+            try:
+                bilibili.download(
+                    'https://www.bilibili.com/video/av16907446/', info_only=True
+                )
+                bilibili.download(
+                    'https://www.bilibili.com/video/av13228063/', info_only=True
+                )
+                return
+            except urllib.error.HTTPError as e:
+                if e.error == 502 and i + 1 < maxRetry:
+                    continue
+                else:
+                    raise e
 
 
 if __name__ == '__main__':

From 93655cf9f4dbca877752a4febe1ba4acc8daf1c1 Mon Sep 17 00:00:00 2001
From: "edward.gao" <scugxl@gmail.com>
Date: Fri, 10 Aug 2018 20:35:55 +0800
Subject: [PATCH 0683/1225] Reduce logging message

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e0ebbac814..96fc60c84b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -358,7 +358,7 @@ def download_video_from_favlist(url, **kwargs):
         vmid = m.group(1)
         favid = m.group(2)
         jsonresult = json.loads(get_content("https://api.bilibili.com/x/space/fav/arc?vmid={}&ps=300&fid={}&order=fav_time&tid=0&keyword=&pn=1&jsonp=jsonp".format(vmid, favid)))
-        print(jsonresult)
+
         # log.wtf("Got files list for vmid" + vmid + " favid:" + favid)
         if jsonresult['code'] != 0:
             log.wtf("Fail to get the files of page " + jsonresult)

From 170dee568bd5882fff8dfae0ff10f35796fc7b57 Mon Sep 17 00:00:00 2001
From: "edward.gao" <scugxl@gmail.com>
Date: Fri, 10 Aug 2018 20:39:25 +0800
Subject: [PATCH 0684/1225] Remove retry for testing bilibili

---
 tests/test.py | 24 ++++++------------------
 1 file changed, 6 insertions(+), 18 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 4a402f1ed7..6562d7cad4 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 
 import unittest
-import urllib
 
 from you_get.extractors import (
     imgur,
@@ -33,23 +32,12 @@ def test_youtube(self):
         )
 
     def test_bilibili(self):
-        maxRetry = 3
-
-        # the bilibi is not stable, it got 502 occasionally
-        for i in range(maxRetry):
-            try:
-                bilibili.download(
-                    'https://www.bilibili.com/video/av16907446/', info_only=True
-                )
-                bilibili.download(
-                    'https://www.bilibili.com/video/av13228063/', info_only=True
-                )
-                return
-            except urllib.error.HTTPError as e:
-                if e.error == 502 and i + 1 < maxRetry:
-                    continue
-                else:
-                    raise e
+        bilibili.download(
+            'https://www.bilibili.com/video/av16907446/', info_only=True
+        )
+        bilibili.download(
+            'https://www.bilibili.com/video/av13228063/', info_only=True
+        )
 
 
 if __name__ == '__main__':

From f80a1dd905f7f91d25568b3344b8e32e0230f6ab Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 10 Aug 2018 16:38:37 +0200
Subject: [PATCH 0685/1225] [youku] I squanch your ccode (close #2611)

---
 src/you_get/extractors/youku.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index bfdb014fad..d5186328fd 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,8 +78,8 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0510'
-        # Found in http://g.alicdn.com/player/ykplayer/0.5.28/youku-player.min.js
+        self.ccode = '0508'
+        # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'
         self.utid = None

From a0b97e56e5658635b5c5f2f32e8b8c14bd35aeb3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 10 Aug 2018 16:40:12 +0200
Subject: [PATCH 0686/1225] version 0.4.1128

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 93a759dec2..76969dc313 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1120'
+__version__ = '0.4.1128'

From bda3b940f42bfec967a00317537d4d92870a66d0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 10 Aug 2018 16:56:51 +0200
Subject: [PATCH 0687/1225] [tests] screw it

---
 tests/test.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 6562d7cad4..047cdb0fa0 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -31,14 +31,5 @@ def test_youtube(self):
             info_only=True
         )
 
-    def test_bilibili(self):
-        bilibili.download(
-            'https://www.bilibili.com/video/av16907446/', info_only=True
-        )
-        bilibili.download(
-            'https://www.bilibili.com/video/av13228063/', info_only=True
-        )
-
-
 if __name__ == '__main__':
     unittest.main()

From 935a4233cda2b506859f51137ab5eed28ddff77c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 15 Aug 2018 13:19:25 +0200
Subject: [PATCH 0688/1225] [youku] get schwifty

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d5186328fd..d7c7938519 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0508'
+        self.ccode = '0511'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 28e1f6cc15568004f8977ab1cb0f3c171cabeeaf Mon Sep 17 00:00:00 2001
From: Mao Chang <1702190+moaix@users.noreply.github.com>
Date: Wed, 15 Aug 2018 22:58:17 +0800
Subject: [PATCH 0689/1225] fix lizhi

---
 src/you_get/extractors/lizhi.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/lizhi.py b/src/you_get/extractors/lizhi.py
index 65988a9f0c..4991df31a2 100644
--- a/src/you_get/extractors/lizhi.py
+++ b/src/you_get/extractors/lizhi.py
@@ -2,8 +2,17 @@
 
 __all__ = ['lizhi_download']
 import json
+import datetime
 from ..common import *
 
+#
+# Worked well but not perfect.
+# TODO: add option --format={sd|hd}
+#
+def get_url(ep):
+    readable = datetime.datetime.fromtimestamp(int(ep['create_time']) / 1000).strftime('%Y/%m/%d')
+    return 'http://cdn5.lizhi.fm/audio/{}/{}_hd.mp3'.format(readable, ep['id'])
+
 # radio_id: e.g. 549759 from http://www.lizhi.fm/549759/
 #
 # Returns a list of tuples (audio_id, title, url) for each episode
@@ -23,7 +32,7 @@ def lizhi_extract_playlist_info(radio_id):
     # (au_cnt), then handle pagination properly.
     api_url = 'http://www.lizhi.fm/api/radio_audios?s=0&l=65535&band=%s' % radio_id
     api_response = json.loads(get_content(api_url))
-    return [(ep['id'], ep['name'], ep['url']) for ep in api_response]
+    return [(ep['id'], ep['name'], get_url(ep)) for ep in api_response]
 
 def lizhi_download_audio(audio_id, title, url, output_dir='.', info_only=False):
     filetype, ext, size = url_info(url)

From 82db2fe8f07e65616d3aff6faf5ec7b61d430534 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 15 Aug 2018 21:42:40 +0200
Subject: [PATCH 0690/1225] [baidu] you got this

---
 src/you_get/extractors/baidu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index b30c9d86ed..65e6209821 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -130,7 +130,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             title = r1(r'title:"([^"]+)"', html)
 
             vhsrc = re.findall(r'"BDE_Image"[^>]+src="([^"]+)"', html) or re.findall(r'vhsrc="([^"]+)"', html)
-            if vhsrc is not None:
+            if len(vhsrc) > 0:
                 ext = 'mp4'
                 size = url_size(vhsrc[0])
                 print_info(site_info, title, ext, size)

From e36404cf2243d8de52062d834f0676a0f95966a7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 25 Aug 2018 15:41:07 +0200
Subject: [PATCH 0691/1225] [youku] fire in the hole!

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d7c7938519..f2e6733607 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0511'
+        self.ccode = '0515'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 2a1bb6978c6e58995e89e055bd3a16042f5c5636 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Aug 2018 14:49:37 +0200
Subject: [PATCH 0692/1225] [twitter] match correct screen_name and item_id in
 a conversation

---
 src/you_get/extractors/twitter.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 8ed400dbb2..1c02797310 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -30,9 +30,9 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         return
 
     html = get_html(url, faker=True)
-    screen_name = r1(r'data-screen-name="([^"]*)"', html) or \
+    screen_name = r1(r'twitter\.com/([^/]+)', url) or r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
-    item_id = r1(r'data-item-id="([^"]*)"', html) or \
+    item_id = r1(r'twitter\.com/[^/]+/status/(\d+)', url) or r1(r'data-item-id="([^"]*)"', html) or \
         r1(r'<meta name="twitter:site:id" content="([^"]*)"', html)
     page_title = "{} [{}]".format(screen_name, item_id)
 

From bea047aa7e0067751f10f40d00b30e5c16442e96 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 29 Aug 2018 16:44:13 +0200
Subject: [PATCH 0693/1225] [[util.os] fix Android termux compatibility (no
 permission to access /proc)

---
 src/you_get/util/os.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/you_get/util/os.py b/src/you_get/util/os.py
index 11730e28c9..1a00d2b5e2 100644
--- a/src/you_get/util/os.py
+++ b/src/you_get/util/os.py
@@ -19,9 +19,11 @@ def detect_os():
     elif 'linux' in syst:
         os = 'linux'
         # detect WSL https://github.com/Microsoft/BashOnWindows/issues/423
-        with open('/proc/version', 'r') as f:
-            if 'microsoft' in f.read().lower():
-                os = 'wsl'
+        try:
+            with open('/proc/version', 'r') as f:
+                if 'microsoft' in f.read().lower():
+                    os = 'wsl'
+        except: pass
     elif 'windows' in syst:
         os = 'windows'
     elif 'bsd' in syst:

From 9ba7690cb9b33a21ec3e068d8b652b2c59d12797 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 29 Aug 2018 17:15:47 +0200
Subject: [PATCH 0694/1225] [bilibili] as you can see

---
 src/you_get/extractors/bilibili.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 96fc60c84b..7234340a78 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -144,11 +144,13 @@ def prepare(self, **kwargs):
         else:
             playinfo = re.search(r'__INITIAL_STATE__=(.*?);\(function\(\)', self.page)
             if playinfo is not None:
-                pages = json.loads(playinfo.group(1))['videoData']['pages']
-                if len(pages) > 1:
-                    qs = dict(parse.parse_qsl(urllib.parse.urlparse(self.url).query))
-                    page = pages[int(qs.get('p', 1)) - 1]
-                    self.title = '{} #{}. {}'.format(self.title, page['page'], page['part'])
+                jsonPlayinfo = json.loads(playinfo.group(1))
+                if 'videoData' in jsonPlayinfo:
+                    pages = jsonPlayinfo['videoData']['pages']
+                    if len(pages) > 1:
+                        qs = dict(parse.parse_qsl(urllib.parse.urlparse(self.url).query))
+                        page = pages[int(qs.get('p', 1)) - 1]
+                        self.title = '{} #{}. {}'.format(self.title, page['page'], page['part'])
 
         if 'bangumi.bilibili.com/movie' in self.url:
             self.movie_entry(**kwargs)

From bd47cb656ec5a7d6ca43ec8664f7c3908b3a8286 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 7 Sep 2018 22:21:24 +0200
Subject: [PATCH 0695/1225] [youtube] fix for new base.js (close #2641)

---
 src/you_get/extractors/youtube.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 33dc470e05..5482f1e400 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -37,6 +37,9 @@ class YouTube(VideoExtractor):
     ]
 
     def decipher(js, s):
+        # Examples:
+        # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
+        # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             code = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', code)
@@ -52,7 +55,8 @@ def tr_js(code):
             return code
 
         js = js.replace('\n', ' ')
-        f1 = match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
+        f1 = match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
+            match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)

From 47b2164b2e21f11de2acbb28303f13f45ddacd6a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 7 Sep 2018 22:26:51 +0200
Subject: [PATCH 0696/1225] [youku] +1

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index f2e6733607..e86b53b9f8 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0515'
+        self.ccode = '0516'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 932fc7a7b5a70e466a11cef0df92aa18aca9d18f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Sep 2018 00:46:15 +0200
Subject: [PATCH 0697/1225] [baidu] as you can see

---
 src/you_get/extractors/baidu.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 65e6209821..a8cb3d5d74 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -129,7 +129,8 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             html = get_html(url)
             title = r1(r'title:"([^"]+)"', html)
 
-            vhsrc = re.findall(r'"BDE_Image"[^>]+src="([^"]+)"', html) or re.findall(r'vhsrc="([^"]+)"', html)
+            vhsrc = re.findall(r'"BDE_Image"[^>]+src="([^"]+\.mp4)"', html) or \
+                re.findall(r'vhsrc="([^"]+)"', html)
             if len(vhsrc) > 0:
                 ext = 'mp4'
                 size = url_size(vhsrc[0])

From a062be55f8eda54f4154870c0dadab4c866cb9cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 8 Sep 2018 00:48:10 +0200
Subject: [PATCH 0698/1225] version 0.4.1140

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 76969dc313..27ed8849f2 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1128'
+__version__ = '0.4.1140'

From 2d8bf0e556fc509fc4dbfc0c884e602962ec837c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 03:08:26 +0200
Subject: [PATCH 0699/1225] [bilibili] there's a solution you're not seeing
 (close #2642)

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 7234340a78..ceeba3ef68 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -115,7 +115,7 @@ def prepare(self, **kwargs):
             self.url = 'http://www.bilibili.com/video/av{}/'.format(aid)
 
         self.ua = fake_headers['User-Agent']
-        self.url = url_locations([self.url])[0]
+        self.url = url_locations([self.url], faker=True)[0]
         frag = urllib.parse.urlparse(self.url).fragment
         # http://www.bilibili.com/video/av3141144/index_2.html#page=3
         if frag:
@@ -125,7 +125,7 @@ def prepare(self, **kwargs):
                 aid = re.search(r'av(\d+)', self.url).group(1)
                 self.url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, page)
         self.referer = self.url
-        self.page = get_content(self.url)
+        self.page = get_content(self.url, headers=fake_headers)
 
         m = re.search(r'<h1.*?>(.*?)</h1>', self.page) or re.search(r'<h1 title="([^"]+)">', self.page)
         if m is not None:

From 3e6387e51c14b9ef6dd437367723dbb8919812ef Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 03:14:53 +0200
Subject: [PATCH 0700/1225] [bilibili] duang duang

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index ceeba3ef68..94eed2ea88 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -381,7 +381,7 @@ def download_video_from_favlist(url, **kwargs):
 
 
 def bilibili_download_playlist_by_url(url, **kwargs):
-    url = url_locations([url])[0]
+    url = url_locations([url], faker=True)[0]
     kwargs['playlist'] = True
     # a bangumi here? possible?
     if 'live.bilibili' in url:

From 89844858199bfc3b3a3317e686e5982c74949777 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 17:31:47 +0200
Subject: [PATCH 0701/1225] [youtube] faster than light

---
 src/you_get/common.py             | 141 ++++++++++++++++--------------
 src/you_get/extractors/youtube.py |  26 +++++-
 2 files changed, 99 insertions(+), 68 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index b19d602f14..d212b62bc2 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -602,7 +602,12 @@ def url_save(
     # the key must be 'Referer' for the hack here
     if refer is not None:
         tmp_headers['Referer'] = refer
-    file_size = url_size(url, faker=faker, headers=tmp_headers)
+    if type(url) is list:
+        file_size = urls_size(url, faker=faker, headers=tmp_headers)
+        is_chunked, urls = True, url
+    else:
+        file_size = url_size(url, faker=faker, headers=tmp_headers)
+        is_chunked, urls = False, [url]
 
     continue_renameing = True
     while continue_renameing:
@@ -655,70 +660,78 @@ def numreturn(a):
     else:
         open_mode = 'wb'
 
-    if received < file_size:
-        if faker:
-            tmp_headers = fake_headers
-        '''
-        if parameter headers passed in, we have it copied as tmp_header
-        elif headers:
-            headers = headers
-        else:
-            headers = {}
-        '''
-        if received:
-            tmp_headers['Range'] = 'bytes=' + str(received) + '-'
-        if refer:
-            tmp_headers['Referer'] = refer
-
-        if timeout:
-            response = urlopen_with_retry(
-                request.Request(url, headers=tmp_headers), timeout=timeout
-            )
-        else:
-            response = urlopen_with_retry(
-                request.Request(url, headers=tmp_headers)
-            )
-        try:
-            range_start = int(
-                response.headers[
-                    'content-range'
-                ][6:].split('/')[0].split('-')[0]
-            )
-            end_length = int(
-                response.headers['content-range'][6:].split('/')[1]
-            )
-            range_length = end_length - range_start
-        except:
-            content_length = response.headers['content-length']
-            range_length = int(content_length) if content_length is not None \
-                else float('inf')
-
-        if file_size != received + range_length:
-            received = 0
-            if bar:
-                bar.received = 0
-            open_mode = 'wb'
-
-        with open(temp_filepath, open_mode) as output:
-            while True:
-                buffer = None
-                try:
-                    buffer = response.read(1024 * 256)
-                except socket.timeout:
-                    pass
-                if not buffer:
-                    if received == file_size:  # Download finished
-                        break
-                    # Unexpected termination. Retry request
-                    tmp_headers['Range'] = 'bytes=' + str(received) + '-'
-                    response = urlopen_with_retry(
-                        request.Request(url, headers=tmp_headers)
-                    )
-                    continue
-                output.write(buffer)
-                received += len(buffer)
+    for url in urls:
+        received_chunk = 0
+        if received < file_size:
+            if faker:
+                tmp_headers = fake_headers
+            '''
+            if parameter headers passed in, we have it copied as tmp_header
+            elif headers:
+                headers = headers
+            else:
+                headers = {}
+            '''
+            if received and not is_chunked:  # only request a range when not chunked
+                tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+            if refer:
+                tmp_headers['Referer'] = refer
+
+            if timeout:
+                response = urlopen_with_retry(
+                    request.Request(url, headers=tmp_headers), timeout=timeout
+                )
+            else:
+                response = urlopen_with_retry(
+                    request.Request(url, headers=tmp_headers)
+                )
+            try:
+                range_start = int(
+                    response.headers[
+                        'content-range'
+                    ][6:].split('/')[0].split('-')[0]
+                )
+                end_length = int(
+                    response.headers['content-range'][6:].split('/')[1]
+                )
+                range_length = end_length - range_start
+            except:
+                content_length = response.headers['content-length']
+                range_length = int(content_length) if content_length is not None \
+                    else float('inf')
+
+            if is_chunked:  # always append if chunked
+                open_mode = 'ab'
+            elif file_size != received + range_length:  # is it ever necessary?
+                received = 0
                 if bar:
-                    bar.update_received(len(buffer))
+                    bar.received = 0
+                open_mode = 'wb'
+
+            with open(temp_filepath, open_mode) as output:
+                while True:
+                    buffer = None
+                    try:
+                        buffer = response.read(1024 * 256)
+                    except socket.timeout:
+                        pass
+                    if not buffer:
+                        if is_chunked and received_chunk == range_length:
+                            break
+                        elif not is_chunked and received == file_size:  # Download finished
+                            break
+                        # Unexpected termination. Retry request
+                        if not is_chunked:  # when
+                            tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+                        response = urlopen_with_retry(
+                            request.Request(url, headers=tmp_headers)
+                        )
+                        continue
+                    output.write(buffer)
+                    received += len(buffer)
+                    received_chunk += len(buffer)
+                    if bar:
+                        bar.update_received(len(buffer))
 
     assert received == os.path.getsize(temp_filepath), '%s == %s == %s' % (
         received, os.path.getsize(temp_filepath), temp_filepath
diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 5482f1e400..1986459015 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -81,6 +81,16 @@ def tr_js(code):
         exec(code, globals(), locals())
         return locals()['sig']
 
+    def chunk_by_range(url, size):
+        urls = []
+        chunk_size = 10485760
+        start, end = 0, chunk_size - 1
+        urls.append('%s&range=%s-%s' % (url, start, end))
+        while end + 1 < size:  # processed size < expected size
+            start, end = end + 1, end + chunk_size
+            urls.append('%s&range=%s-%s' % (url, start, end))
+        return urls
+
     def get_url_from_vid(vid):
         return 'https://youtu.be/{}'.format(vid)
 
@@ -290,13 +300,15 @@ def prepare(self, **kwargs):
                         if not dash_size:
                             try: dash_size = url_size(dash_url)
                             except: continue
+                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                        dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
                         self.dash_streams[itag] = {
                             'quality': '%sx%s' % (w, h),
                             'itag': itag,
                             'type': mimeType,
                             'mime': mimeType,
                             'container': 'mp4',
-                            'src': [dash_url, dash_mp4_a_url],
+                            'src': [dash_urls, dash_mp4_a_urls],
                             'size': int(dash_size) + int(dash_mp4_a_size)
                         }
                 elif mimeType == 'video/webm':
@@ -310,13 +322,15 @@ def prepare(self, **kwargs):
                         if not dash_size:
                             try: dash_size = url_size(dash_url)
                             except: continue
+                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                        dash_webm_a_urls = self.__class__.chunk_by_range(dash_webm_a_url, int(dash_webm_a_size))
                         self.dash_streams[itag] = {
                             'quality': '%sx%s' % (w, h),
                             'itag': itag,
                             'type': mimeType,
                             'mime': mimeType,
                             'container': 'webm',
-                            'src': [dash_url, dash_webm_a_url],
+                            'src': [dash_urls, dash_webm_a_urls],
                             'size': int(dash_size) + int(dash_webm_a_size)
                         }
         except:
@@ -353,13 +367,15 @@ def prepare(self, **kwargs):
                                 dash_url += '&signature={}'.format(sig)
                             dash_size = stream['clen']
                             itag = stream['itag']
+                            dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                            dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
                             self.dash_streams[itag] = {
                                 'quality': stream['size'],
                                 'itag': itag,
                                 'type': mimeType,
                                 'mime': mimeType,
                                 'container': 'mp4',
-                                'src': [dash_url, dash_mp4_a_url],
+                                'src': [dash_urls, dash_mp4_a_urls],
                                 'size': int(dash_size) + int(dash_mp4_a_size)
                             }
                         elif stream['type'].startswith('video/webm'):
@@ -378,13 +394,15 @@ def prepare(self, **kwargs):
                             except UnboundLocalError as e:
                                 audio_url = dash_mp4_a_url
                                 audio_size = int(dash_mp4_a_size)
+                            dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                            audio_urls = self.__class__.chunk_by_range(audio_url, int(audio_size))
                             self.dash_streams[itag] = {
                                 'quality': stream['size'],
                                 'itag': itag,
                                 'type': mimeType,
                                 'mime': mimeType,
                                 'container': 'webm',
-                                'src': [dash_url, audio_url],
+                                'src': [dash_urls, audio_urls],
                                 'size': int(dash_size) + int(audio_size)
                             }
 

From f8c39fbe4cbe83d8c1f316d3c221808dbfc22931 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 23:18:39 +0200
Subject: [PATCH 0702/1225] [common] post_content: allow post_data_raw

---
 src/you_get/common.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index d212b62bc2..5ce5299016 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -439,7 +439,7 @@ def get_content(url, headers={}, decoded=True):
     return data
 
 
-def post_content(url, headers={}, post_data={}, decoded=True):
+def post_content(url, headers={}, post_data={}, decoded=True, **kwargs):
     """Post the content of a URL via sending a HTTP POST request.
 
     Args:
@@ -457,7 +457,10 @@ def post_content(url, headers={}, post_data={}, decoded=True):
     if cookies:
         cookies.add_cookie_header(req)
         req.headers.update(req.unredirected_hdrs)
-    post_data_enc = bytes(parse.urlencode(post_data), 'utf-8')
+    if kwargs.get('post_data_raw'):
+        post_data_enc = bytes(kwargs['post_data_raw'], 'utf-8')
+    else:
+        post_data_enc = bytes(parse.urlencode(post_data), 'utf-8')
     response = urlopen_with_retry(req, data=post_data_enc)
     data = response.read()
 

From f3cb2512a32f5fd14e91f0cded96cb5677a1b7fa Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 23:23:50 +0200
Subject: [PATCH 0703/1225] [tumblr] take my consent

---
 src/you_get/extractors/tumblr.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index fe4973bed0..f01c335201 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -13,7 +13,29 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         universal_download(url, output_dir, merge=merge, info_only=info_only)
         return
 
-    html = parse.unquote(get_html(url)).replace('\/', '/')
+    import ssl
+    ssl_context = request.HTTPSHandler(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
+    cookie_handler = request.HTTPCookieProcessor()
+    opener = request.build_opener(ssl_context, cookie_handler)
+    request.install_opener(opener)
+
+    page = get_html(url)
+    form_key = match1(page, r'id="tumblr_form_key" content="([^"]+)"')
+    if form_key is not None:
+        # bypass GDPR consent page
+        referer = 'https://www.tumblr.com/privacy/consent?redirect=%s' % parse.quote_plus(url)
+        post_content('https://www.tumblr.com/svc/privacy/consent',
+                     headers={
+                         'Content-Type': 'application/json',
+                         'User-Agent': fake_headers['User-Agent'],
+                         'Referer': referer,
+                         'X-tumblr-form-key': form_key,
+                         'X-Requested-With': 'XMLHttpRequest'
+                     },
+                     post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url)
+        page = get_html(url)
+
+    html = parse.unquote(page).replace('\/', '/')
     feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)
 
     if feed in ['photo', 'photoset', 'entry'] or feed is None:

From cc69f0945aaaff6535af020d84effa7d3c89ffab Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 23:44:51 +0200
Subject: [PATCH 0704/1225] [universal] let Pinterest go suck a lemon

---
 src/you_get/extractors/universal.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 57b9b2d10b..43272cb812 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -106,6 +106,9 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 title = '%s' % i
                 i += 1
 
+            if r1(r'(https://pinterest.com/pin/)', url):
+                continue
+
             candies.append({'url': url,
                             'title': title})
 

From 5dfee49688645497061841d981c50fe577e4ba32 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 23:49:39 +0200
Subject: [PATCH 0705/1225] [common] post_content: make log right

---
 src/you_get/common.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5ce5299016..f40b022066 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -450,8 +450,10 @@ def post_content(url, headers={}, post_data={}, decoded=True, **kwargs):
     Returns:
         The content as a string.
     """
-
-    logging.debug('post_content: %s \n post_data: %s' % (url, post_data))
+    if kwargs.get('post_data_raw'):
+        logging.debug('post_content: %s\npost_data_raw: %s' % (url, kwargs['post_data_raw']))
+    else:
+        logging.debug('post_content: %s\npost_data: %s' % (url, post_data))
 
     req = request.Request(url, headers=headers)
     if cookies:

From 76e831d443e9ca8e4344ad0ab5130ffc25eb9a73 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Sep 2018 23:51:43 +0200
Subject: [PATCH 0706/1225] version 0.4.1148

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 27ed8849f2..4f5b1645ae 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1140'
+__version__ = '0.4.1148'

From fcdfce68d29d2f56b552fb3883f41fea7b7bf9de Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 16 Sep 2018 14:18:31 +0200
Subject: [PATCH 0707/1225] [tumblr] squanch this

---
 src/you_get/extractors/tumblr.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index f01c335201..bc37fa433f 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -33,7 +33,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
                          'X-Requested-With': 'XMLHttpRequest'
                      },
                      post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url)
-        page = get_html(url)
+        page = get_html(url, faker=True)
 
     html = parse.unquote(page).replace('\/', '/')
     feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)
@@ -43,9 +43,9 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         page_title = r1(r'<meta name="description" content="([^"\n]+)', html) or \
                      r1(r'<meta property="og:description" content="([^"\n]+)', html) or \
                      r1(r'<title>([^<\n]*)', html)
-        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.jpg)', html) +\
-               re.findall(r'(https?://[^;"&]+/tumblr_[^;"]+_\d+\.png)', html) +\
-               re.findall(r'(https?://[^;"&]+/tumblr_[^";]+_\d+\.gif)', html)
+        urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.jpg)', html) +\
+               re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.png)', html) +\
+               re.findall(r'(https?://[^;"&]+/tumblr_[^";&]+_\d+\.gif)', html)
 
         tuggles = {}
         for url in urls:

From caabb083f7c78f9170347ed0d4f60330c26b6da9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 16 Sep 2018 14:34:48 +0200
Subject: [PATCH 0708/1225] [tumblr] squanch that

---
 src/you_get/extractors/tumblr.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index bc37fa433f..d63aee728d 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -121,11 +121,15 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         r1(r'<meta property="og:description" content="([^"]*)" />', html) or
         r1(r'<title>([^<\n]*)', html) or url.split("/")[4]).replace('\n', '')
 
-    type, ext, size = url_info(real_url)
+    # this is better
+    vcode = r1(r'tumblr_(\w+)', real_url)
+    real_url = 'https://vt.media.tumblr.com/tumblr_%s.mp4' % vcode
+
+    type, ext, size = url_info(real_url, faker=True)
 
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([real_url], title, ext, size, output_dir, merge = merge)
+        download_urls([real_url], title, ext, size, output_dir, merge=merge)
 
 site_info = "Tumblr.com"
 download = tumblr_download

From 1ea4abdb779ce2073accad10fd795a4add418142 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 17 Sep 2018 13:52:04 +0200
Subject: [PATCH 0709/1225] [universal] lalalala

---
 src/you_get/extractors/universal.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 43272cb812..a1ab153624 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -67,9 +67,9 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^ ;"\'\\]+' + i + r'[^ ;"\'\\]*)', page)
+            urls += re.findall(r'(https?://[^ ;&"\'\\]+' + i + r'[^ ;&"\'\\]*)', page)
 
-            p_urls = re.findall(r'(https?%3A%2F%2F[^;&]+' + i + r'[^;&]*)', page)
+            p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page)
             urls += [parse.unquote(url) for url in p_urls]
 
             q_urls = re.findall(r'(https?:\\\\/\\\\/[^ ;"\']+' + i + r'[^ ;"\']*)', page)

From fc8df5eb24b0856d4b0a7c99bebb58b901cb40f1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 17 Sep 2018 13:52:44 +0200
Subject: [PATCH 0710/1225] [naver] call universal_download if video extraction
 fails

---
 src/you_get/extractors/naver.py | 42 ++++++++++++++++++---------------
 1 file changed, 23 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
index d79e5245e0..b9eef8d72e 100644
--- a/src/you_get/extractors/naver.py
+++ b/src/you_get/extractors/naver.py
@@ -7,31 +7,35 @@
 
 from ..util import log
 from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size
+from .universal import *
 
 __all__ = ['naver_download_by_url']
 
 
-def naver_download_by_url(url, info_only=False, **kwargs):
+def naver_download_by_url(url, output_dir='.', merge=True, info_only=False, **kwargs):
     ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'
     page = get_content(url)
-    og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1)
-    params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
-    vid = params_dict['vid'][0]
-    key = params_dict['outKey'][0]
-    meta_str = get_content(ep.format(vid, key))
-    meta_json = json.loads(meta_str)
-    if 'errorCode' in meta_json:
-        log.wtf(meta_json['errorCode'])
-    title = meta_json['meta']['subject']
-    videos = meta_json['videos']['list']
-    video_list = sorted(videos, key=lambda video: video['encodingOption']['width'])
-    video_url = video_list[-1]['source']
-    # size = video_list[-1]['size']
-    # result wrong size
-    size = url_size(video_url)
-    print_info(site_info, title, 'mp4', size)
-    if not info_only:
-        download_urls([video_url], title, 'mp4', size, **kwargs)
+    try:
+        og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1)
+        params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
+        vid = params_dict['vid'][0]
+        key = params_dict['outKey'][0]
+        meta_str = get_content(ep.format(vid, key))
+        meta_json = json.loads(meta_str)
+        if 'errorCode' in meta_json:
+            log.wtf(meta_json['errorCode'])
+        title = meta_json['meta']['subject']
+        videos = meta_json['videos']['list']
+        video_list = sorted(videos, key=lambda video: video['encodingOption']['width'])
+        video_url = video_list[-1]['source']
+        # size = video_list[-1]['size']
+        # result wrong size
+        size = url_size(video_url)
+        print_info(site_info, title, 'mp4', size)
+        if not info_only:
+            download_urls([video_url], title, 'mp4', size, **kwargs)
+    except:
+        universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
 
 site_info = "naver.com"
 download = naver_download_by_url

From 286a7788d2fc57b4888185eb795ab8ab7e6d5ca6 Mon Sep 17 00:00:00 2001
From: Mateusz Piotrowski <0mp@FreeBSD.org>
Date: Wed, 19 Sep 2018 09:38:26 +0200
Subject: [PATCH 0711/1225] Add installation instructions for FreeBSD

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index f6f8efdcac..1450057786 100644
--- a/README.md
+++ b/README.md
@@ -113,6 +113,14 @@ You can install `you-get` easily via:
 $ brew install you-get
 ```
 
+### Option 8: pkg (FreeBSD only)
+
+You can install `you-get` easily via:
+
+```
+# pkg install you-get
+```
+
 ### Shell completion
 
 Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](https://github.com/soimort/you-get/tree/develop/contrib/completion). Please consult your shell's manual for how to take advantage of them.

From 4b7753f2d43701203d82d8826966ef34f2bd29e8 Mon Sep 17 00:00:00 2001
From: mq-liu <mingquan_liu@163.com>
Date: Fri, 21 Sep 2018 15:02:22 +0800
Subject: [PATCH 0712/1225] update sohu.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

原来的下载, 下载特别慢, 还会出错
---
 src/you_get/extractors/sohu.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 58eb1ac76f..a8c81be57a 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -15,9 +15,9 @@
         new api
 '''
 
-def real_url(host,vid,tvid,new,clipURL,ck):
-    url = 'http://'+host+'/?prot=9&prod=flash&pt=1&file='+clipURL+'&new='+new +'&key='+ ck+'&vid='+str(vid)+'&uid='+str(int(time.time()*1000))+'&t='+str(random())+'&rb=1'
-    return json.loads(get_html(url))['url']
+def real_url(fileName,key,ch):
+    url = "https://data.vod.itc.cn/ip?new=" + fileName + "&num=1&key=" + key + "&ch=" + ch + "&pt=1&pg=2&prod=h5n"
+    return json.loads(get_html(url))['servers'][0]['url']
 
 def sohu_download(url, output_dir = '.', merge = True, info_only = False, extractor_proxy=None, **kwargs):
     if re.match(r'http://share.vrs.sohu.com', url):
@@ -51,9 +51,8 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         title = data['tvName']
         size = sum(data['clipsBytes'])
         assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
-        for new,clip,ck, in zip(data['su'], data['clipsURL'], data['ck']):
-            clipURL = urlparse(clip).path
-            urls.append(real_url(host,hqvid,tvid,new,clipURL,ck))
+        for fileName,key,ch, in zip(data['su'], data['ck'], data['ch']):
+            urls.append(real_url(fileName,key,ch))
         # assert data['clipsURL'][0].endswith('.mp4')
 
     else:
@@ -66,9 +65,8 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         title = data['tvName']
         size = sum(map(int,data['clipsBytes']))
         assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
-        for new,clip,ck, in zip(data['su'], data['clipsURL'], data['ck']):
-            clipURL = urlparse(clip).path
-            urls.append(real_url(host,vid,tvid,new,clipURL,ck))
+        for fileName,key,ch, in zip(data['su'], data['ck'], data['ch']):
+            urls.append(real_url(fileName,key,ch))
 
     print_info(site_info, title, 'mp4', size)
     if not info_only:

From 251b80962a29aabc314580b8d30887d88fff007c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 14 Oct 2018 15:34:49 +0200
Subject: [PATCH 0713/1225] util.fs: \t not allowed in FAT (close #2646)

---
 src/you_get/util/fs.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/util/fs.py b/src/you_get/util/fs.py
index b6b7069afa..c04a10a74d 100644
--- a/src/you_get/util/fs.py
+++ b/src/you_get/util/fs.py
@@ -13,6 +13,7 @@ def legitimize(text, os=detect_os()):
         ord('|'): '-',
     })
 
+    # FIXME: do some filesystem detection
     if os == 'windows' or os == 'cygwin' or os == 'wsl':
         # Windows (non-POSIX namespace)
         text = text.translate({
@@ -28,6 +29,7 @@ def legitimize(text, os=detect_os()):
             ord('>'): '-',
             ord('['): '(',
             ord(']'): ')',
+            ord('\t'): ' ',
         })
     else:
         # *nix

From f94c8d530df77254e3597dbd16a4ba40b2048e56 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 17 Oct 2018 22:26:55 +0200
Subject: [PATCH 0714/1225] util.log: add yes_or_no()

---
 src/you_get/util/log.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
index a2c77ab50b..67b26b7815 100644
--- a/src/you_get/util/log.py
+++ b/src/you_get/util/log.py
@@ -96,3 +96,9 @@ def wtf(message, exit_code=1):
     print_log(message, RED, BOLD)
     if exit_code is not None:
         sys.exit(exit_code)
+
+def yes_or_no(message):
+    ans = str(input('%s (y/N) ' % message)).lower().strip()
+    if ans == 'y':
+        return True
+    return False

From 5026436e8a573a3a7656184738dfe6a537936291 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 17 Oct 2018 22:28:21 +0200
Subject: [PATCH 0715/1225] common: add proper warning and confirming before
 overwriting things

---
 src/you_get/common.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f40b022066..88e7d8d354 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -622,7 +622,7 @@ def url_save(
                 if not is_part:
                     if bar:
                         bar.done()
-                    print(
+                    log.w(
                         'Skipping {}: file already exists'.format(
                             tr(os.path.basename(filepath))
                         )
@@ -648,7 +648,10 @@ def numreturn(a):
                         print('Changing name to %s' % tr(os.path.basename(filepath)), '...')
                         continue_renameing = True
                         continue
-                    print('Overwriting %s' % tr(os.path.basename(filepath)), '...')
+                    if log.yes_or_no('File with this name already exists. Overwrite?'):
+                        log.w('Overwriting %s ...' % tr(os.path.basename(filepath)))
+                    else:
+                        return
         elif not os.path.exists(os.path.dirname(filepath)):
             os.mkdir(os.path.dirname(filepath))
 
@@ -925,7 +928,7 @@ def download_urls(
     if total_size:
         if not force and os.path.exists(output_filepath) and not auto_rename\
                 and os.path.getsize(output_filepath) >= total_size * 0.9:
-            print('Skipping %s: file already exists' % output_filepath)
+            log.w('Skipping %s: file already exists' % output_filepath)
             print()
             return
         bar = SimpleProgressBar(total_size, len(urls))

From fabb35a5b982d918e94abe89cd5a63a501b518cb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 20 Oct 2018 16:22:14 +0200
Subject: [PATCH 0716/1225] [sohu] do not zip ch

---
 src/you_get/extractors/sohu.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index a8c81be57a..607bf44c6b 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -15,7 +15,7 @@
         new api
 '''
 
-def real_url(fileName,key,ch):
+def real_url(fileName, key, ch):
     url = "https://data.vod.itc.cn/ip?new=" + fileName + "&num=1&key=" + key + "&ch=" + ch + "&pt=1&pg=2&prod=h5n"
     return json.loads(get_html(url))['servers'][0]['url']
 
@@ -51,8 +51,8 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         title = data['tvName']
         size = sum(data['clipsBytes'])
         assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
-        for fileName,key,ch, in zip(data['su'], data['ck'], data['ch']):
-            urls.append(real_url(fileName,key,ch))
+        for fileName, key in zip(data['su'], data['ck']):
+            urls.append(real_url(fileName, key, data['ch']))
         # assert data['clipsURL'][0].endswith('.mp4')
 
     else:
@@ -65,8 +65,8 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         title = data['tvName']
         size = sum(map(int,data['clipsBytes']))
         assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
-        for fileName,key,ch, in zip(data['su'], data['ck'], data['ch']):
-            urls.append(real_url(fileName,key,ch))
+        for fileName, key in zip(data['su'], data['ck']):
+            urls.append(real_url(fileName, key, data['ch']))
 
     print_info(site_info, title, 'mp4', size)
     if not info_only:

From 50b66f3151dc63ffb7b7e216056906afff150358 Mon Sep 17 00:00:00 2001
From: kxy000 <kxy000@qq.com>
Date: Mon, 22 Oct 2018 23:54:10 +0800
Subject: [PATCH 0717/1225] Update pptv.py

add user agent
---
 src/you_get/extractors/pptv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index 17503c1cd4..8d95a5a133 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -192,14 +192,14 @@ def prepare(self, **kwargs):
         if self.url and not self.vid:
             if not re.match(r'http://v.pptv.com/show/(\w+)\.html', self.url):
                 raise('Unknown url pattern')
-            page_content = get_content(self.url)
+            page_content = get_content(self.url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"})
             self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)')
 
         if not self.vid:
             raise('Cannot find id')
         api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid)
         api_url += '?appplt=flp&appid=pptv.flashplayer.vod&appver=3.4.2.28&type=&version=4'
-        dom = parseString(get_content(api_url))
+        dom = parseString(get_content(api_url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"}))
         self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom)
         xml_streams = merge_meta(m_items, m_streams, m_segs)
         for stream_id in xml_streams:

From dc1581869fe819e127bd17da83fccf5fc08d1339 Mon Sep 17 00:00:00 2001
From: beyond <yangbing@gozap.com>
Date: Thu, 25 Oct 2018 11:12:36 +0800
Subject: [PATCH 0718/1225] Update miapai api

---
 src/you_get/extractors/yixia.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/yixia.py b/src/you_get/extractors/yixia.py
index ff45730d66..d3d1ef350e 100644
--- a/src/you_get/extractors/yixia.py
+++ b/src/you_get/extractors/yixia.py
@@ -7,6 +7,24 @@
 from json import loads
 import re
 
+#----------------------------------------------------------------------
+def miaopai_download_by_smid(smid, output_dir = '.', merge = True, info_only = False):
+    """"""
+    api_endpoint = 'https://n.miaopai.com/api/aj_media/info.json?smid={smid}'.format(smid = smid)
+
+    html = get_content(api_endpoint)
+
+    api_content = loads(html)
+
+    video_url = api_content['data']['meta_data'][0]['play_urls']['l']
+    title = api_content['data']['description']
+
+    type, ext, size = url_info(video_url)
+
+    print_info(site_info, title, type, size)
+    if not info_only:
+        download_urls([video_url], title, ext, size, output_dir, merge=merge)
+
 #----------------------------------------------------------------------
 def yixia_miaopai_download_by_scid(scid, output_dir = '.', merge = True, info_only = False):
     """"""
@@ -47,7 +65,11 @@ def yixia_xiaokaxiu_download_by_scid(scid, output_dir = '.', merge = True, info_
 def yixia_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     """wrapper"""
     hostname = urlparse(url).hostname
-    if 'miaopai.com' in hostname:  #Miaopai
+    if 'n.miaopai.com' == hostname: 
+        smid = match1(url, r'n\.miaopai\.com/media/([^.]+)') 
+        miaopai_download_by_smid(smid, output_dir, merge, info_only)
+        return
+    elif 'miaopai.com' in hostname:  #Miaopai
         yixia_download_by_scid = yixia_miaopai_download_by_scid
         site_info = "Yixia Miaopai"
 

From 035294e573b9397bbe2278e1666c54268562e7e0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Oct 2018 22:36:24 +0200
Subject: [PATCH 0719/1225] [bilibili] the production of too many useful things
 results in

---
 src/you_get/extractors/bilibili.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 94eed2ea88..ed9663c0d4 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -192,7 +192,12 @@ def entry(self, **kwargs):
             index_id = int(re.search(r'index_(\d+)', self.url).group(1))
             cid = page_list[index_id-1]['cid'] # change cid match rule
         except:
-            cid = re.search(r'"cid":(\d+)', self.page).group(1)
+            page = re.search(r'p=(\d+)', self.url)
+            if page is None:
+                p = 1
+            else:
+                p = int(page.group(1))
+            cid = re.search(r'"cid":(\d+),"page":%s' % p, self.page).group(1)
         if cid is not None:
             self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs)
         else:

From 389b55b12229ebe114120b6ccd1490446b75fdb4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Oct 2018 22:48:04 +0200
Subject: [PATCH 0720/1225] .travis.yml: skip more of flake8

---
 .travis.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 9df327b0e4..7e772c8c2e 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,9 +8,10 @@ python:
   - "3.6"
   - "nightly"
   - "pypy3"
-before_install: pip install flake8
+before_install:
+  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then pip install flake8; fi
 before_script:
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi
 script: make test
 sudo: false
 notifications:

From 6d6c219a282c1887483c2a167735f802b8686467 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 28 Oct 2018 13:33:28 +0100
Subject: [PATCH 0721/1225] version 0.4.1164

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 4f5b1645ae..e1a5349dba 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1148'
+__version__ = '0.4.1164'

From 1f70be6aa937a6533b3e990334d4138283949deb Mon Sep 17 00:00:00 2001
From: lc4t <lc4t0.0@gmail.com>
Date: Thu, 1 Nov 2018 11:57:21 +0800
Subject: [PATCH 0722/1225] use new bilibili live api, fix None Content-Type

---
 src/you_get/common.py              | 2 +-
 src/you_get/extractors/bilibili.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 88e7d8d354..3d04e8a16e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -429,7 +429,7 @@ def get_content(url, headers={}, decoded=True):
     # Decode the response body
     if decoded:
         charset = match1(
-            response.getheader('Content-Type'), r'charset=([\w-]+)'
+            response.getheader('Content-Type', ''), r'charset=([\w-]+)'
         )
         if charset is not None:
             data = data.decode(charset)
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index ed9663c0d4..71cc7fc2d4 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -22,7 +22,7 @@
 
 class Bilibili(VideoExtractor):
     name = 'Bilibili'
-    live_api = 'http://live.bilibili.com/api/playurl?cid={}&otype=json'
+    live_api = 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid={}&quality=0&platform=web'
     api_url = 'http://interface.bilibili.com/v2/playurl?'
     bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
     live_room_init_api_url = 'https://api.live.bilibili.com/room/v1/Room/room_init?id={}'
@@ -233,7 +233,7 @@ def live_entry(self, **kwargs):
 
         api_url = self.live_api.format(self.room_id)
         json_data = json.loads(get_content(api_url))
-        urls = [json_data['durl'][0]['url']]
+        urls = [json_data['data']['durl'][0]['url']]
 
         self.streams['live'] = {}
         self.streams['live']['src'] = urls

From aa221f137817d4b30611dea8860b6c625cc3f5ee Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 7 Nov 2018 16:49:18 +0100
Subject: [PATCH 0723/1225] [youtube] whatever this
 (0,window.encodeURIComponent) thing is (fix #2652)

---
 src/you_get/extractors/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 1986459015..b1a680b9c5 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -40,6 +40,7 @@ def decipher(js, s):
         # Examples:
         # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
         # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
+        # - https://www.youtube.com/yts/jsbin/player-vfls4aurX/da_DK/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             code = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', code)
@@ -55,7 +56,8 @@ def tr_js(code):
             return code
 
         js = js.replace('\n', ' ')
-        f1 = match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
+        f1 = match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
+            match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
             match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))

From 046dcea5e805ff18473a6ecdd722ddfa437855fd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 7 Nov 2018 16:59:58 +0100
Subject: [PATCH 0724/1225] version 0.4.1167

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e1a5349dba..883b7dca9e 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1164'
+__version__ = '0.4.1167'

From 7d9ce6b8d01145a4d8215916d2acab29d2d08565 Mon Sep 17 00:00:00 2001
From: Yingdong Yang <storm-yyd@outlook.com>
Date: Fri, 9 Nov 2018 13:36:03 +0800
Subject: [PATCH 0725/1225] fix iwara

---
 src/you_get/extractors/iwara.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 50d14fb8d8..a30159d748 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -17,20 +17,20 @@
 
 def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     global headers
-    video_hash=match1(url, r'http://\w+.iwara.tv/videos/(\w+)')
-    video_url=match1(url, r'(http://\w+.iwara.tv)/videos/\w+')
-    html = get_content(url,headers=headers)
+    video_hash = match1(url, r'https?://\w+.iwara.tv/videos/(\w+)')
+    video_url = match1(url, r'(https?://\w+.iwara.tv)/videos/\w+')
+    html = get_content(url, headers=headers)
     title = r1(r'<title>(.*)</title>', html)
-    api_url=video_url+'/api/video/'+video_hash
-    content=get_content(api_url,headers=headers)
-    data=json.loads(content)
-    type,ext,size=url_info(data[0]['uri'], headers=headers)
-    down_urls=data[0]['uri']
-    print_info(down_urls,title+data[0]['resolution'],type,size)
+    api_url = video_url + '/api/video/' + video_hash
+    content = get_content(api_url, headers=headers)
+    data = json.loads(content)
+    down_urls = 'https:' + data[0]['uri']
+    type, ext, size = url_info(down_urls, headers=headers)
+    print_info(site_info, title+data[0]['resolution'], type, size)
 
     if not info_only:
-        download_urls([down_urls], title, ext, size, output_dir, merge = merge,headers=headers)
+        download_urls([down_urls], title, ext, size, output_dir, merge=merge, headers=headers)
 
-site_info = "iwara"
+site_info = "Iwara"
 download = iwara_download
 download_playlist = playlist_not_supported('iwara')

From 64e15159abc326dbe3afb16883e652942361cb5b Mon Sep 17 00:00:00 2001
From: Vcinly <vcinly@gmail.com>
Date: Sun, 11 Nov 2018 13:03:29 +0800
Subject: [PATCH 0726/1225] support download bilibili uploader all videos

---
 src/you_get/extractors/bilibili.py | 34 ++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 71cc7fc2d4..053b4d199f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -384,6 +384,38 @@ def download_video_from_favlist(url, **kwargs):
     else:
         log.wtf("Fail to parse the fav title" + url, "")
 
+def download_video_from_totallist(url, page, **kwargs):
+    # the url has format: https://space.bilibili.com/64169458/#/video
+    m = re.search(r'space\.bilibili\.com/(\d+)/.*?video', url)
+    mid = ""
+    if m is not None:
+        mid = m.group(1)
+        jsonresult = json.loads(get_content("https://space.bilibili.com/ajax/member/getSubmitVideos?mid={}&pagesize=100&tid=0&page={}&keyword=&order=pubdate&jsonp=jsonp".format(mid, page)))
+        if jsonresult['status']:
+            videos = jsonresult['data']['vlist']
+            videocount = len(videos)
+            for i in range(videocount):
+                videoid = videos[i]["aid"]
+                videotitle = videos[i]["title"]
+                videourl = "https://www.bilibili.com/video/av{}".format(videoid)
+                print("Start downloading ", videotitle, " video ", videotitle)
+                kwargs["output_dir"] = kwargs["output_dir"] + '/' + str(videoid)
+                download_cover(videos[i]['pic'], videotitle, **kwargs)
+                Bilibili().download_by_url(videourl, subtitle=videotitle, **kwargs)
+            if page <= jsonresult['pages']:
+                page += 1
+                download_video_from_totallist(url, page, **kwargs)
+        else:
+            log.wtf("Fail to get the files of page " + jsonresult)
+            sys.exit(2)
+
+    else:
+        log.wtf("Fail to parse the fav title" + url, "")
+
+def download_cover(url, title, **kwargs):
+    if re.match(r'https?://', url) is None:
+        url = 'https:' + url
+    download_urls([url], title, "jpg", 0, kwargs["output_dir"])
 
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url], faker=True)[0]
@@ -403,6 +435,8 @@ def bilibili_download_playlist_by_url(url, **kwargs):
     elif 'favlist' in url:
         # this a fav list folder
         download_video_from_favlist(url, **kwargs)
+    elif 'video' in url:
+        download_video_from_totallist(url, 1, **kwargs)
     else:
         aid = re.search(r'av(\d+)', url).group(1)
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))

From 470b74f3aff77168e0d11c4f7caa470bb1cac238 Mon Sep 17 00:00:00 2001
From: Vcinly <vcinly@gmail.com>
Date: Sat, 17 Nov 2018 22:44:25 +0800
Subject: [PATCH 0727/1225] [bilibili] fixed space videos url detect

---
 src/you_get/extractors/bilibili.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 053b4d199f..10077bf883 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -399,10 +399,8 @@ def download_video_from_totallist(url, page, **kwargs):
                 videotitle = videos[i]["title"]
                 videourl = "https://www.bilibili.com/video/av{}".format(videoid)
                 print("Start downloading ", videotitle, " video ", videotitle)
-                kwargs["output_dir"] = kwargs["output_dir"] + '/' + str(videoid)
-                download_cover(videos[i]['pic'], videotitle, **kwargs)
                 Bilibili().download_by_url(videourl, subtitle=videotitle, **kwargs)
-            if page <= jsonresult['pages']:
+            if page < jsonresult['data']['pages']:
                 page += 1
                 download_video_from_totallist(url, page, **kwargs)
         else:
@@ -410,12 +408,7 @@ def download_video_from_totallist(url, page, **kwargs):
             sys.exit(2)
 
     else:
-        log.wtf("Fail to parse the fav title" + url, "")
-
-def download_cover(url, title, **kwargs):
-    if re.match(r'https?://', url) is None:
-        url = 'https:' + url
-    download_urls([url], title, "jpg", 0, kwargs["output_dir"])
+        log.wtf("Fail to parse the video title" + url, "")
 
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url], faker=True)[0]
@@ -435,7 +428,7 @@ def bilibili_download_playlist_by_url(url, **kwargs):
     elif 'favlist' in url:
         # this a fav list folder
         download_video_from_favlist(url, **kwargs)
-    elif 'video' in url:
+    elif re.match(r'https?://space.bilibili.com/\d+/#/video', url):
         download_video_from_totallist(url, 1, **kwargs)
     else:
         aid = re.search(r'av(\d+)', url).group(1)

From 9f68d3c37a5c98e75cd884332f92fd27d6246c82 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 18 Nov 2018 23:50:38 +0100
Subject: [PATCH 0728/1225] [bilibili] fix bangumi thing

---
 src/you_get/extractors/bilibili.py | 19 +++----------------
 1 file changed, 3 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 10077bf883..9ae5464048 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -265,22 +265,9 @@ def bangumi_entry(self, **kwargs):
             episode_id = frag
         else:
             episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page) or re.search(r'\/ep(\d+)', self.url).group(1)
-        # cont = post_content('http://bangumi.bilibili.com/web_api/get_source', post_data=dict(episode_id=episode_id))
-        # cid = json.loads(cont)['result']['cid']
-        cont = get_content('http://bangumi.bilibili.com/web_api/episode/{}.json'.format(episode_id))
-        ep_info = json.loads(cont)['result']['currentEpisode']
-
-        bangumi_data = get_bangumi_info(str(ep_info['seasonId']))
-        bangumi_payment = bangumi_data.get('payment')
-        if bangumi_payment and bangumi_payment['price'] != '0':
-            log.w("It's a paid item")
-        # ep_ids = collect_bangumi_epids(bangumi_data)
-
-        index_title = ep_info['indexTitle']
-        long_title = ep_info['longTitle'].strip()
-        cid = ep_info['danmaku']
-
-        self.title = '{} [{} {}]'.format(self.title, index_title, long_title)
+        data = json.loads(re.search(r'__INITIAL_STATE__=(.+);\(function', self.page).group(1))
+        cid = data['epInfo']['cid']
+        # index_title = data['epInfo']['index_title']
         self.download_by_vid(cid, bangumi=True, **kwargs)
 
 

From 98d9580dd344b36c65e59652292b63552ddf21cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 19 Nov 2018 00:14:04 +0100
Subject: [PATCH 0729/1225] [common] fix google_search

---
 src/you_get/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 3d04e8a16e..6a239154cb 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1575,9 +1575,9 @@ def google_search(url):
     url = 'https://www.google.com/search?tbm=vid&q=%s' % parse.quote(keywords)
     page = get_content(url, headers=fake_headers)
     videos = re.findall(
-        r'<a href="(https?://[^"]+)" onmousedown="[^"]+">([^<]+)<', page
+        r'<a href="(https?://[^"]+)" onmousedown="[^"]+"><h3 class="[^"]*">([^<]+)<', page
     )
-    vdurs = re.findall(r'<span class="vdur _dwc">([^<]+)<', page)
+    vdurs = re.findall(r'<span class="vdur[^"]*">([^<]+)<', page)
     durs = [r1(r'(\d+:\d+)', unescape_html(dur)) for dur in vdurs]
     print('Google Videos search:')
     for v in zip(videos, durs):

From e14f21f323c5210ab2f04a0a861d1515c2178092 Mon Sep 17 00:00:00 2001
From: bitdust <github@bitdust.me>
Date: Wed, 21 Nov 2018 01:56:57 +0800
Subject: [PATCH 0730/1225] fix bilibili title regex

match '<span>' with html attribute
---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9ae5464048..079501c615 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -130,7 +130,7 @@ def prepare(self, **kwargs):
         m = re.search(r'<h1.*?>(.*?)</h1>', self.page) or re.search(r'<h1 title="([^"]+)">', self.page)
         if m is not None:
             self.title = m.group(1)
-            s = re.search(r'<span>([^<]+)</span>', m.group(1))
+            s = re.search(r'<span.*?>([^<]+)</span>', m.group(1))
             if s:
                 self.title = unescape_html(s.group(1))
         if self.title is None:

From 5946a545751ae8376beec54032ea92e2fc6e710d Mon Sep 17 00:00:00 2001
From: FengLi666 <fengli97@hotmail.com>
Date: Thu, 22 Nov 2018 13:45:00 +0800
Subject: [PATCH 0731/1225] fix acfun bangumi page

---
 src/you_get/extractors/acfun.py | 35 +++++++++++++++++++++++----------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 4b45c5e962..772132fe6c 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -105,27 +105,42 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
             pass
 
 def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    assert re.match(r'http://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url)
-    html = get_content(url)
+    assert re.match(r'http://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
+
+    if re.match(r'http://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
+        html = get_content(url)
+        title = r1(r'data-title="([^"]+)"', html)
+        if match1(url, r'_(\d+)$'):  # current P
+            title = title + " " + r1(r'active">([^<]*)', html)
+        vid = r1('data-vid="(\d+)"', html)
+        up = r1('data-name="([^"]+)"', html)
+    # bangumi
+    elif re.match("http://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
+        html = get_content(url)
+        title = match1(html, r'"newTitle"\s*:\s*"([^"]+)"')
+        if match1(url, r'_(\d+)$'):  # current P
+            title = title + " " + r1(r'active">([^<]*)', html)
+        vid = match1(html, r'videoId="(\d+)"')
+        up = "acfun"
+    else:
+        raise NotImplemented
 
-    title = r1(r'data-title="([^"]+)"', html)
+    assert title and vid
     title = unescape_html(title)
     title = escape_file_path(title)
-    assert title
-    if match1(url, r'_(\d+)$'): # current P
-        title = title + " " + r1(r'active">([^<]*)', html)
-
-    vid = r1('data-vid="(\d+)"', html)
-    up = r1('data-name="([^"]+)"', html)
     p_title = r1('active">([^<]+)', html)
     title = '%s (%s)' % (title, up)
-    if p_title: title = '%s - %s' % (title, p_title)
+    if p_title:
+        title = '%s - %s' % (title, p_title)
+
+
     acfun_download_by_vid(vid, title,
                           output_dir=output_dir,
                           merge=merge,
                           info_only=info_only,
                           **kwargs)
 
+
 site_info = "AcFun.tv"
 download = acfun_download
 download_playlist = playlist_not_supported('acfun')

From ab8a3a2ccff0292f38fa8f229b4cfb8784d6bcd6 Mon Sep 17 00:00:00 2001
From: URenko <18209292+URenko@users.noreply.github.com>
Date: Sun, 25 Nov 2018 20:07:52 +0800
Subject: [PATCH 0732/1225] fix acfun flv support

---
 src/you_get/extractors/acfun.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 772132fe6c..200a3f542b 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -85,9 +85,13 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
             _, _, seg_size = url_info(url)
             size += seg_size
 #fallback to flvhd is not quite possible
-        print_info(site_info, title, 'mp4', size)
+        if re.search(r'fid=[0-9A-Z\-]*.flv', preferred[0][0]):
+            ext = 'flv'
+        else:
+            ext = 'mp4'
+        print_info(site_info, title, ext, size)
         if not info_only:
-            download_urls(preferred[0], title, 'mp4', size, output_dir=output_dir, merge=merge)
+            download_urls(preferred[0], title, ext, size, output_dir=output_dir, merge=merge)
     else:
         raise NotImplementedError(sourceType)
 

From 1f52bd01ae062c2b51511aa76cd56c939dc0e02d Mon Sep 17 00:00:00 2001
From: astronaut <519537870@qq.com>
Date: Mon, 26 Nov 2018 20:57:46 +0800
Subject: [PATCH 0733/1225] support bilibili audio

---
 src/you_get/extractor.py           |  2 +-
 src/you_get/extractors/bilibili.py | 78 +++++++++++++++++++++++++++++-
 2 files changed, 78 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 4c9ccaa58c..8aeed560f0 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -211,7 +211,7 @@ def download(self, **kwargs):
                 ext = self.dash_streams[stream_id]['container']
                 total_size = self.dash_streams[stream_id]['size']
 
-            if ext == 'm3u8':
+            if ext == 'm3u8' or ext == 'm4a':
                 ext = 'mp4'
 
             if not urls:
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 079501c615..24821d77fc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -137,7 +137,6 @@ def prepare(self, **kwargs):
             m = re.search(r'property="og:title" content="([^"]+)"', self.page)
             if m is not None:
                 self.title = m.group(1)
-
         if 'subtitle' in kwargs:
             subtitle = kwargs['subtitle']
             self.title = '{} {}'.format(self.title, subtitle)
@@ -162,6 +161,8 @@ def prepare(self, **kwargs):
             self.live_entry(**kwargs)
         elif 'vc.bilibili.com' in self.url:
             self.vc_entry(**kwargs)
+        elif 'audio/au' in self.url:
+            self.audio_entry(**kwargs)
         else:
             self.entry(**kwargs)
 
@@ -173,6 +174,30 @@ def movie_entry(self, **kwargs):
         self.title = page_list[0]['pagename']
         self.download_by_vid(page_list[0]['cid'], True, bangumi_movie=True, **kwargs)
 
+    def audio_entry(self, **kwargs):
+        assert re.match(r'https?://www.bilibili.com/audio/au\d+', self.url)
+        patt = r"(\d+)"
+        audio_id = re.search(patt, self.url).group(1)
+        audio_info_url = \
+            'https://www.bilibili.com/audio/music-service-c/web/song/info?sid={}'.format(audio_id)
+        audio_info_response = json.loads(get_content(audio_info_url))
+        if audio_info_response['msg'] != 'success':
+            log.wtf('fetch audio information failed!')
+            sys.exit(2)
+        self.title = audio_info_response['data']['title']
+        # TODO:there is no quality option for now
+        audio_download_url = \
+            'https://www.bilibili.com/audio/music-service-c/web/url?sid={}&privilege=2&quality=2'.format(audio_id)
+        audio_download_response = json.loads(get_content(audio_download_url))
+        if audio_download_response['msg'] != 'success':
+            log.wtf('fetch audio resource failed!')
+            sys.exit(2)
+        self.streams['mp4'] = {}
+        self.streams['mp4']['src'] = [audio_download_response['data']['cdns'][0]]
+        self.streams['mp4']['container'] = 'm4a'
+        self.streams['mp4']['size'] = audio_download_response['data']['size']
+
+
     def entry(self, **kwargs):
         # tencent player
         tc_flashvars = re.search(r'"bili-cid=\d+&bili-aid=\d+&vid=([^"]+)"', self.page)
@@ -370,6 +395,29 @@ def download_video_from_favlist(url, **kwargs):
 
     else:
         log.wtf("Fail to parse the fav title" + url, "")
+def download_music_from_favlist(url, page, **kwargs):
+    m = re.search(r'https?://www.bilibili.com/audio/mycollection/(\d+)', url)
+    if m is not None:
+        sid = m.group(1)
+        json_result = json.loads(get_content("https://www.bilibili.com/audio/music-service-c/web/song/of-coll?"
+                                             "sid={}&pn={}&ps=100".format(sid, page)))
+        if json_result['msg'] == 'success':
+            music_list = json_result['data']['data']
+            music_count = len(music_list)
+            for i in range(music_count):
+                audio_id = music_list[i]['id']
+                audio_title = music_list[i]['title']
+                audio_url = "https://www.bilibili.com/audio/au{}".format(audio_id)
+                print("Start downloading music ", audio_title)
+                Bilibili().download_by_url(audio_url, **kwargs)
+            if page < json_result['data']['pageCount']:
+                page += 1
+                download_music_from_favlist(url, page, **kwargs)
+        else:
+            log.wtf("Fail to get music list of page " + json_result)
+            sys.exit(2)
+    else:
+        log.wtf("Fail to parse the sid from " + url, "")
 
 def download_video_from_totallist(url, page, **kwargs):
     # the url has format: https://space.bilibili.com/64169458/#/video
@@ -397,6 +445,30 @@ def download_video_from_totallist(url, page, **kwargs):
     else:
         log.wtf("Fail to parse the video title" + url, "")
 
+def download_music_from_totallist(url, page, **kwargs):
+    m = re.search(r'https?://www.bilibili.com/audio/am(\d+)\?type=\d', url)
+    if m is not None:
+        sid = m.group(1)
+        json_result = json.loads(get_content("https://www.bilibili.com/audio/music-service-c/web/song/of-menu?"
+                                             "sid={}&pn={}&ps=100".format(sid, page)))
+        if json_result['msg'] == 'success':
+            music_list = json_result['data']['data']
+            music_count = len(music_list)
+            for i in range(music_count):
+                audio_id = music_list[i]['id']
+                audio_title = music_list[i]['title']
+                audio_url = "https://www.bilibili.com/audio/au{}".format(audio_id)
+                print("Start downloading music ",audio_title)
+                Bilibili().download_by_url(audio_url, **kwargs)
+            if page < json_result['data']['pageCount']:
+                page += 1
+                download_music_from_totallist(url, page, **kwargs)
+        else:
+            log.wtf("Fail to get music list of page " + json_result)
+            sys.exit(2)
+    else:
+        log.wtf("Fail to parse the sid from " + url, "")
+
 def bilibili_download_playlist_by_url(url, **kwargs):
     url = url_locations([url], faker=True)[0]
     kwargs['playlist'] = True
@@ -417,6 +489,10 @@ def bilibili_download_playlist_by_url(url, **kwargs):
         download_video_from_favlist(url, **kwargs)
     elif re.match(r'https?://space.bilibili.com/\d+/#/video', url):
         download_video_from_totallist(url, 1, **kwargs)
+    elif re.match(r'https://www.bilibili.com/audio/mycollection/\d+', url):
+        download_music_from_favlist(url, 1, **kwargs)
+    elif re.match(r'https?://www.bilibili.com/audio/am\d+\?type=\d', url):
+        download_music_from_totallist(url, 1, **kwargs)
     else:
         aid = re.search(r'av(\d+)', url).group(1)
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))

From 32a2e24785e835a790754eb58f3eaaf024db056a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 14:12:43 +0100
Subject: [PATCH 0734/1225] [youku] sometimes naive

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index e86b53b9f8..75a49c707e 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0516'
+        self.ccode = '0590'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From fdb021371487955318fdec7b94cb48f483c90f76 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 17:36:59 +0100
Subject: [PATCH 0735/1225] [youtube] don't fail the whole playlist

---
 src/you_get/extractors/youtube.py | 23 ++++++++++++++++-------
 1 file changed, 16 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index b1a680b9c5..bc1bc4692d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -144,7 +144,10 @@ def download_playlist_by_url(self, url, **kwargs):
         for video in videos:
             vid = parse_query_param(video, 'v')
             index = parse_query_param(video, 'index')
-            self.__class__().download_by_url(self.__class__.get_url_from_vid(vid), index=index, **kwargs)
+            try:
+                self.__class__().download_by_url(self.__class__.get_url_from_vid(vid), index=index, **kwargs)
+            except:
+                pass
 
     def prepare(self, **kwargs):
         assert self.url or self.vid
@@ -160,7 +163,8 @@ def prepare(self, **kwargs):
 
         ytplayer_config = None
         if 'status' not in video_info:
-            log.wtf('[Failed] Unknown status.')
+            log.wtf('[Failed] Unknown status.', exit_code=None)
+            raise
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
                 self.title = parse.unquote_plus(video_info['title'][0])
@@ -192,7 +196,8 @@ def prepare(self, **kwargs):
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+});ytplayer', video_page).group(1))
                 except:
                     msg = re.search('class="message">([^<]+)<', video_page).group(1)
-                    log.wtf('[Failed] "%s"' % msg.strip())
+                    log.wtf('[Failed] "%s"' % msg.strip(), exit_code=None)
+                    raise
 
                 if 'title' in ytplayer_config['args']:
                     # 150 Restricted from playback on certain sites
@@ -201,18 +206,22 @@ def prepare(self, **kwargs):
                     self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 else:
-                    log.wtf('[Error] The uploader has not made this video available in your country.')
+                    log.wtf('[Error] The uploader has not made this video available in your country.', exit_code=None)
+                    raise
                     #self.title = re.search('<meta name="title" content="([^"]+)"', video_page).group(1)
                     #stream_list = []
 
             elif video_info['errorcode'] == ['100']:
-                log.wtf('[Failed] This video does not exist.', exit_code=int(video_info['errorcode'][0]))
+                log.wtf('[Failed] This video does not exist.', exit_code=None) #int(video_info['errorcode'][0])
+                raise
 
             else:
-                log.wtf('[Failed] %s' % video_info['reason'][0], exit_code=int(video_info['errorcode'][0]))
+                log.wtf('[Failed] %s' % video_info['reason'][0], exit_code=None) #int(video_info['errorcode'][0])
+                raise
 
         else:
-            log.wtf('[Failed] Invalid status.')
+            log.wtf('[Failed] Invalid status.', exit_code=None)
+            raise
 
         # YouTube Live
         if ytplayer_config and (ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1'):

From 85598839c07422f2cd4bae3a579c6650e416aeec Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 18:29:22 +0100
Subject: [PATCH 0736/1225] [tiktok] new site support

---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  3 ++-
 src/you_get/extractors/tiktok.py   | 21 +++++++++++++++++++++
 3 files changed, 24 insertions(+), 1 deletion(-)
 create mode 100644 src/you_get/extractors/tiktok.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6a239154cb..478939109b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -102,6 +102,7 @@
     'soundcloud'       : 'soundcloud',
     'ted'              : 'ted',
     'theplatform'      : 'theplatform',
+    'tiktok'           : 'tiktok',
     'tucao'            : 'tucao',
     'tudou'            : 'tudou',
     'tumblr'           : 'tumblr',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 649a911f36..302433c0b6 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -67,6 +67,7 @@
 from .soundcloud import *
 from .suntv import *
 from .theplatform import *
+from .tiktok import *
 from .tucao import *
 from .tudou import *
 from .tumblr import *
@@ -88,4 +89,4 @@
 from .khan import *
 from .zhanqi import *
 from .kuaishou import *
-from .zhibo import *
\ No newline at end of file
+from .zhibo import *
diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
new file mode 100644
index 0000000000..9718abde03
--- /dev/null
+++ b/src/you_get/extractors/tiktok.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+__all__ = ['tiktok_download']
+
+from ..common import *
+
+def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    html = get_html(url)
+    title = r1(r'<title>(.*?)</title>', html)
+    dataText = r1(r'var data = \[(.*)\] ', html)
+    data = json.loads(dataText)
+    source = 'http:' + data['video']['play_addr']['url_list'][0]
+    mime, ext, size = url_info(source)
+
+    print_info(site_info, title, mime, size)
+    if not info_only:
+        download_urls([source], title, ext, size, output_dir, merge=merge)
+
+site_info = "TikTok.com"
+download = tiktok_download
+download_playlist = playlist_not_supported('tiktok')

From 5fece0bd1cb1e68f19993e024bab968de2778d83 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 18:54:05 +0100
Subject: [PATCH 0737/1225] [tiktok] more URL patterns

---
 src/you_get/extractors/tiktok.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 9718abde03..e9ff63abd0 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -7,7 +7,9 @@
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
     title = r1(r'<title>(.*?)</title>', html)
-    dataText = r1(r'var data = \[(.*)\] ', html)
+    video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
+    title = '%s [%s]' % (title, video_id)
+    dataText = r1(r'var data = \[(.*)\] ', html) or r1(r'var data = (\{.*\})', html)
     data = json.loads(dataText)
     source = 'http:' + data['video']['play_addr']['url_list'][0]
     mime, ext, size = url_info(source)

From 0e90b9b00053e178eab032909fa8f1af16a55f90 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 21:51:11 +0100
Subject: [PATCH 0738/1225] version 0.4.1181

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 883b7dca9e..e0a79a8e5a 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1167'
+__version__ = '0.4.1181'

From fe3eeacd543f2850f47cc9cbe8efe425129c3084 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 22:01:26 +0100
Subject: [PATCH 0739/1225] claim to support Python 3.7

---
 you-get.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/you-get.json b/you-get.json
index 594742c21a..56f8212af2 100644
--- a/you-get.json
+++ b/you-get.json
@@ -25,6 +25,7 @@
     "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Multimedia",

From 0930e0784e59a049ae82d29f4a44ad8471cbd622 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 22:07:04 +0100
Subject: [PATCH 0740/1225] update README.md

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 1450057786..856f6c809c 100644
--- a/README.md
+++ b/README.md
@@ -424,6 +424,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
+| TikTok | <https://www.tiktok.com/>      |✓| | |
 | 中国体育(TV) | <http://v.zhibo.tv/> </br><http://video.zhibo.tv/>    |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.

From 265818d39bcb1afae231ad108f4492b022ca9bbc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 30 Nov 2018 22:08:38 +0100
Subject: [PATCH 0741/1225] .travis.yml: add 3.7-dev

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 7e772c8c2e..c11cbe34d2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,6 +6,7 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7-dev"
   - "nightly"
   - "pypy3"
 before_install:

From 45f951b9b3267279f53a956454010decbbdef0ae Mon Sep 17 00:00:00 2001
From: FengLi666 <fengli97@hotmail.com>
Date: Sat, 1 Dec 2018 22:34:49 +0800
Subject: [PATCH 0742/1225] fix bilibili bangumi page

delete out of date regex which causes error
---
 src/you_get/extractors/bilibili.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 24821d77fc..5ed7f28d05 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -284,12 +284,6 @@ def vc_entry(self, **kwargs):
         self.streams['vc']['size'] = int(item['video_size'])
 
     def bangumi_entry(self, **kwargs):
-        bangumi_id = re.search(r'(\d+)', self.url).group(1)
-        frag = urllib.parse.urlparse(self.url).fragment
-        if frag:
-            episode_id = frag
-        else:
-            episode_id = re.search(r'first_ep_id\s*=\s*"(\d+)"', self.page) or re.search(r'\/ep(\d+)', self.url).group(1)
         data = json.loads(re.search(r'__INITIAL_STATE__=(.+);\(function', self.page).group(1))
         cid = data['epInfo']['cid']
         # index_title = data['epInfo']['index_title']

From e37291dfd86a3cb6bf780585ab51cd308bf8de26 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Dec 2018 14:11:59 +0100
Subject: [PATCH 0743/1225] [instagram] hey ho

---
 src/you_get/extractors/instagram.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 65fc01f510..6537b60662 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -29,7 +29,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 image_url = edge['node']['display_url']
                 if 'video_url' in edge['node']:
                     image_url = edge['node']['video_url']
-                image_url = image_url.split('?')[0]
+                image_url = image_url.split('?')[0] # (not here: '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net')
                 ext = image_url.split('.')[-1]
                 size = int(get_head(image_url)['Content-Length'])
                 print_info(site_info, title, ext, size)
@@ -44,7 +44,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
             if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
                 image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
-            image_url = image_url.split('?')[0]
+            image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
             ext = image_url.split('.')[-1]
             size = int(get_head(image_url)['Content-Length'])
             print_info(site_info, title, ext, size)

From 926c7b283a640532044731d2b5ba6d700bb81702 Mon Sep 17 00:00:00 2001
From: perror <15058342792@163.com>
Date: Thu, 6 Dec 2018 00:55:08 +0800
Subject: [PATCH 0744/1225] fix ixigua downloading failure

---
 src/you_get/extractors/ixigua.py | 128 +++++++++++++++++++++++++++++--
 1 file changed, 123 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 5913344292..3cf07b094e 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -1,14 +1,132 @@
 #!/usr/bin/env python
-__all__ = ['ixigua_download']
+import base64
 
-from .toutiao import download as toutiao_download
-from .toutiao import download_playlist as toutiao_download_playlist
+import binascii
+
+from ..common import *
+import random
+import ctypes
+from json import loads
+
+__all__ = ['ixigua_download', 'ixigua_download_playlist_by_url']
+
+headers = {
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 "
+                  "Safari/537.36",
+}
+
+
+def int_overflow(val):
+    maxint = 2147483647
+    if not -maxint - 1 <= val <= maxint:
+        val = (val + (maxint + 1)) % (2 * (maxint + 1)) - maxint - 1
+    return val
+
+
+def unsigned_right_shitf(n, i):
+    if n < 0:
+        n = ctypes.c_uint32(n).value
+    if i < 0:
+        return -int_overflow(n << abs(i))
+    return int_overflow(n >> i)
+
+
+def get_video_url_from_video_id(video_id):
+    """Splicing URLs according to video ID to get video details"""
+    # from js
+    data = [""] * 256
+    for index, _ in enumerate(data):
+        t = index
+        for i in range(8):
+            t = -306674912 ^ unsigned_right_shitf(t, 1) if 1 & t else unsigned_right_shitf(t, 1)
+        data[index] = t
+
+    def tmp():
+        rand_num = random.random()
+        path = "/video/urls/v/1/toutiao/mp4/{video_id}?r={random_num}".format(video_id=video_id,
+                                                                              random_num=str(rand_num)[2:])
+        e = o = r = -1
+        i, a = 0, len(path)
+        while i < a:
+            e = ord(path[i])
+            i += 1
+            if e < 128:
+                r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ e)]
+            else:
+                if e < 2048:
+                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (192 | e >> 6 & 31))]
+                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]
+                else:
+                    if 55296 <= e < 57344:
+                        e = (1023 & e) + 64
+                        i += 1
+                        o = 1023 & t.url(i)
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (240 | e >> 8 & 7))]
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 2 & 63))]
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | o >> 6 & 15 | (3 & e) << 4))]
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & o))]
+                    else:
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (224 | e >> 12 & 15))]
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 6 & 63))]
+                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]
+
+        return "https://ib.365yg.com{path}&s={param}".format(path=path, param=unsigned_right_shitf(r ^ -1, 0))
+
+    while 1:
+        url = tmp()
+        if url.split("=")[-1][0] != "-":  # 参数s不能为负数
+            return url
 
 
 def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    return toutiao_download(url.replace('ixigua', '365yg'))
+    # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
+    html = get_html(url, faker=True)
+    video_id = match1(html, r"videoId\s*:\s*'([^']+)'")
+    title = match1(html, r"title: '(\S+)',")
+    if not video_id:
+        log.e("video_id not found, url:{}".format(url))
+        return
+    video_info_url = get_video_url_from_video_id(video_id)
+    video_info = loads(get_content(video_info_url))
+    if video_info.get("code", 1) != 0:
+        log.e("Get video info from {} error: server return code {}".format(video_info_url, video_info.get("code", 1)))
+        return
+    if not video_info.get("data", None):
+        log.e("Get video info from {} error: The server returns JSON value"
+              " without data or data is empty".format(video_info_url))
+        return
+    if not video_info["data"].get("video_list", None):
+        log.e("Get video info from {} error: The server returns JSON value"
+              " without data.video_list or data.video_list is empty".format(video_info_url))
+        return
+    if not video_info["data"]["video_list"].get("video_1", None):
+        log.e("Get video info from {} error: The server returns JSON value"
+              " without data.video_list.video_1 or data.video_list.video_1 is empty".format(video_info_url))
+        return
+    size = int(video_info["data"]["video_list"]["video_1"]["size"])
+    print_info(site_info=site_info, title=title, type="mp4", size=size)  # 该网站只有mp4类型文件
+    if not info_only:
+        video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"].encode("utf-8"))
+        download_urls([video_url.decode("utf-8")], title, "mp4", size, output_dir, merge=merge, headers=headers, **kwargs)
+
+
+def ixigua_download_playlist_by_url(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    assert "user" in url, "Only support users to publish video list,Please provide a similar url:" \
+                          "https://www.ixigua.com/c/user/6907091136/"
+
+    user_id = url.split("/")[-2] if url[-1] == "/" else url.split("/")[-1]
+    params = {"max_behot_time": "0", "max_repin_time": "0", "count": "20", "page_type": "0", "user_id": user_id}
+    while 1:
+        url = "https://www.ixigua.com/c/user/article/?" + "&".join(["{}={}".format(k, v) for k, v in params.items()])
+        video_list = loads(get_content(url, headers=headers))
+        params["max_behot_time"] = video_list["next"]["max_behot_time"]
+        for video in video_list["data"]:
+            ixigua_download("https://www.ixigua.com/i{}/".format(video["item_id"]), output_dir, merge, info_only,
+                            **kwargs)
+        if video_list["next"]["max_behot_time"] == 0:
+            break
 
 
 site_info = "ixigua.com"
 download = ixigua_download
-download_playlist = toutiao_download_playlist
+download_playlist = ixigua_download_playlist_by_url

From 58e806d72e57f919d3a4f9fd6a30c9691fa46903 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Dec 2018 23:24:24 +0100
Subject: [PATCH 0745/1225] [youtube] use prefix to avoid potential namespace
 conflict (fix #2666)

---
 src/you_get/extractors/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index bc1bc4692d..9f2d286353 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -62,7 +62,7 @@ def tr_js(code):
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
-        f1def = 'function %s%s' % (f1, f1def)
+        f1def = 'function main_%s%s' % (f1, f1def)  # prefix to avoid potential namespace conflict
         code = tr_js(f1def)
         f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
         for f2 in f2s:
@@ -79,7 +79,7 @@ def tr_js(code):
 
         f1 = re.sub(r'(as|if|in|is|or)', r'_\1', f1)
         f1 = re.sub(r'\$', '_dollar', f1)
-        code = code + 'sig=%s(s)' % f1
+        code = code + 'sig=main_%s(s)' % f1  # prefix to avoid potential namespace conflict
         exec(code, globals(), locals())
         return locals()['sig']
 

From 7dbfece21ffbe586ae08731a0f2a86e0882c38ad Mon Sep 17 00:00:00 2001
From: lniwn <lniwn@live.com>
Date: Thu, 6 Dec 2018 21:24:10 +0800
Subject: [PATCH 0746/1225] [miaopai] fix weibo.com download error

---
 src/you_get/extractors/miaopai.py | 52 ++++++++++++++++++++++++++++++-
 1 file changed, 51 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index f37d45b004..b827024b72 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -2,9 +2,12 @@
 
 __all__ = ['miaopai_download']
 
+import string
+import random
 from ..common import *
 import urllib.error
 import urllib.parse
+from ..util import fs
 
 fake_headers_mobile = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
@@ -20,6 +23,10 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
 
     mobile_page = get_content(page_url, headers=fake_headers_mobile)
     url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
+    if url is None:
+        wb_mp = re.search(r'<script src=([\'"])(.+?wb_mp\.js)\1>', mobile_page).group(2)
+        return miaopai_download_by_wbmp(wb_mp, fid, output_dir=output_dir, merge=merge,
+                                        info_only=info_only, total_size=None, **kwargs)
     title = match1(mobile_page, r'<title>((.|\n)+?)</title>')
     if not title:
         title = fid
@@ -29,7 +36,47 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
     if not info_only:
         download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
-#----------------------------------------------------------------------
+
+def miaopai_download_by_wbmp(wbmp_url, fid, info_only=False, **kwargs):
+    headers = {}
+    headers.update(fake_headers_mobile)
+    headers['Host'] = 'imgaliyuncdn.miaopai.com'
+    wbmp = get_content(wbmp_url, headers=headers)
+    appid = re.search(r'appid:\s*?([^,]+?),', wbmp).group(1)
+    jsonp = re.search(r'jsonp:\s*?([\'"])(\w+?)\1', wbmp).group(2)
+    population = [i for i in string.ascii_lowercase] + [i for i in string.digits]
+    info_url = '{}?{}'.format('http://p.weibo.com/aj_media/info', parse.urlencode({
+        'appid': appid.strip(),
+        'fid': fid,
+        jsonp.strip(): '_jsonp' + ''.join(random.sample(population, 11))
+    }))
+    headers['Host'] = 'p.weibo.com'
+    jsonp_text = get_content(info_url, headers=headers)
+    jsonp_dict = json.loads(match1(jsonp_text, r'\(({.+})\)'))
+    if jsonp_dict['code'] != 200:
+        log.wtf('[Failed] "%s"' % jsonp_dict['msg'])
+    video_url = jsonp_dict['data']['meta_data'][0]['play_urls']['l']
+    title = jsonp_dict['data']['description']
+    title = title.replace('\n', '_')
+    ext = 'mp4'
+    headers['Host'] = 'f.us.sinaimg.cn'
+    print_info(site_info, title, ext, url_info(video_url, headers=headers)[2])
+    if not info_only:
+        download_urls([video_url], fs.legitimize(title), ext, headers=headers, **kwargs)
+
+
+def miaopai_download_direct(url, info_only, **kwargs):
+    mobile_page = get_content(url, headers=fake_headers_mobile)
+    title = re.search(r'([\'"])title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    title = title.replace('\n', '_')
+    stream_url = re.search(r'([\'"])stream_url\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    ext = 'mp4'
+    print_info(site_info, title, ext, url_info(stream_url, headers=fake_headers_mobile)[2])
+    if not info_only:
+        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
+
+
+# ----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
@@ -37,6 +84,8 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
     elif '/p/230444' in url:
         fid = match1(url, r'/p/230444(\w+)')
         miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
+    elif re.match(r'^http[s]://weibo\.com/\d+/.+', url):
+        miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
     else:
         mobile_page = get_content(url, headers = fake_headers_mobile)
         hit = re.search(r'"page_url"\s*:\s*"([^"]+)"', mobile_page)
@@ -46,6 +95,7 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
             escaped_url = hit.group(1)
             miaopai_download(urllib.parse.unquote(escaped_url), output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
+
 site_info = "miaopai"
 download = miaopai_download
 download_playlist = playlist_not_supported('miaopai')

From 8454f878d20100c6bcac1f1ad99d765d9cebcbb9 Mon Sep 17 00:00:00 2001
From: perror <15058342792@163.com>
Date: Fri, 7 Dec 2018 21:38:39 +0800
Subject: [PATCH 0747/1225] [zhihu] Add zhihu.com support

---
 README.md                          |  1 +
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  1 +
 src/you_get/extractors/zhihu.py    | 79 ++++++++++++++++++++++++++++++
 4 files changed, 82 insertions(+)
 create mode 100644 src/you_get/extractors/zhihu.py

diff --git a/README.md b/README.md
index 856f6c809c..f524c60d92 100644
--- a/README.md
+++ b/README.md
@@ -426,6 +426,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 抖音 | <https://www.douyin.com/>      |✓| | |
 | TikTok | <https://www.tiktok.com/>      |✓| | |
 | 中国体育(TV) | <http://v.zhibo.tv/> </br><http://video.zhibo.tv/>    |✓| | |
+| 知乎 | <https://www.zhihu.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
 
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 478939109b..7818216303 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -128,6 +128,7 @@
     'youtube'          : 'youtube',
     'zhanqi'           : 'zhanqi',
     'zhibo'            : 'zhibo',
+    'zhihu'            : 'zhihu',
 }
 
 dry_run = False
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 302433c0b6..d2c4c7b718 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -90,3 +90,4 @@
 from .zhanqi import *
 from .kuaishou import *
 from .zhibo import *
+from .zhihu import *
diff --git a/src/you_get/extractors/zhihu.py b/src/you_get/extractors/zhihu.py
new file mode 100644
index 0000000000..64f81423ca
--- /dev/null
+++ b/src/you_get/extractors/zhihu.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+__all__ = ['zhihu_download', 'zhihu_download_playlist']
+
+from ..common import *
+import json
+
+
+def zhihu_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    paths = url.split("/")
+    # question or column
+    if len(paths) < 3 and len(paths) < 6:
+        raise TypeError("URL does not conform to specifications, Support column and question only."
+                        "Example URL: https://zhuanlan.zhihu.com/p/51669862 or "
+                        "https://www.zhihu.com/question/267782048/answer/490720324")
+
+    if ("question" not in paths or "answer" not in paths) and "zhuanlan.zhihu.com" not in paths:
+        raise TypeError("URL does not conform to specifications, Support column and question only."
+                        "Example URL: https://zhuanlan.zhihu.com/p/51669862 or "
+                        "https://www.zhihu.com/question/267782048/answer/490720324")
+
+    html = get_html(url, faker=True)
+    title = match1(html, r'data-react-helmet="true">(.*?)</title>')
+    for index, video_id in enumerate(matchall(html, [r'<a class="video-box" href="\S+video/(\d+)"'])):
+        try:
+            video_info = json.loads(
+                get_content(r"https://lens.zhihu.com/api/videos/{}".format(video_id), headers=fake_headers))
+        except json.decoder.JSONDecodeError:
+            log.w("Video id not found:{}".format(video_id))
+            continue
+
+        play_list = video_info["playlist"]
+        # first High Definition
+        # second Second Standard Definition
+        # third ld. What is ld ?
+        # finally continue
+        data = play_list.get("hd", play_list.get("sd", play_list.get("ld", None)))
+        if not data:
+            log.w("Video id No play address:{}".format(video_id))
+            continue
+        print_info(site_info, title, data["format"], data["size"])
+        if not info_only:
+            ext = "_{}.{}".format(index, data["format"])
+            if kwargs.get("zhihu_offset"):
+                ext = "_{}".format(kwargs["zhihu_offset"]) + ext
+            download_urls([data["play_url"]], title, ext, data["size"],
+                          output_dir=output_dir, merge=merge, **kwargs)
+
+
+def zhihu_download_playlist(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if "question" not in url or "answer" in url:  # question page
+        raise TypeError("URL does not conform to specifications, Support question only."
+                        " Example URL: https://www.zhihu.com/question/267782048")
+    url = url.split("?")[0]
+    if url[-1] == "/":
+        question_id = url.split("/")[-2]
+    else:
+        question_id = url.split("/")[-1]
+    videos_url = r"https://www.zhihu.com/api/v4/questions/{}/answers".format(question_id)
+    try:
+        questions = json.loads(get_content(videos_url))
+    except json.decoder.JSONDecodeError:
+        raise TypeError("Check whether the problem URL exists.Example URL: https://www.zhihu.com/question/267782048")
+
+    count = 0
+    while 1:
+        for data in questions["data"]:
+            kwargs["zhihu_offset"] = count
+            zhihu_download("https://www.zhihu.com/question/{}/answer/{}".format(question_id, data["id"]),
+                           output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+            count += 1
+        if questions["paging"]["is_end"]:
+            return
+        questions = json.loads(get_content(questions["paging"]["next"], headers=fake_headers))
+
+
+site_info = "zhihu.com"
+download = zhihu_download
+download_playlist = zhihu_download_playlist

From c90a44c586908f81206384f07da0624d9e28b611 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 10 Dec 2018 17:22:52 +0100
Subject: [PATCH 0748/1225] [instagram] let's go

---
 src/you_get/extractors/instagram.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 6537b60662..9dd7207d63 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -29,9 +29,14 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 image_url = edge['node']['display_url']
                 if 'video_url' in edge['node']:
                     image_url = edge['node']['video_url']
-                image_url = image_url.split('?')[0] # (not here: '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net')
-                ext = image_url.split('.')[-1]
-                size = int(get_head(image_url)['Content-Length'])
+                try:
+                    image_url = image_url.split('?')[0]
+                    ext = image_url.split('.')[-1]
+                    size = int(get_head(image_url)['Content-Length'])
+                except:
+                    image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
+                    ext = image_url.split('.')[-1]
+                    size = int(get_head(image_url)['Content-Length'])
                 print_info(site_info, title, ext, size)
                 if not info_only:
                     download_urls(urls=[image_url],
@@ -44,9 +49,14 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
             if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
                 image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
-            image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
-            ext = image_url.split('.')[-1]
-            size = int(get_head(image_url)['Content-Length'])
+            try:
+                image_url = image_url.split('?')[0]
+                ext = image_url.split('.')[-1]
+                size = int(get_head(image_url)['Content-Length'])
+            except:
+                image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
+                ext = image_url.split('.')[-1]
+                size = int(get_head(image_url)['Content-Length'])
             print_info(site_info, title, ext, size)
             if not info_only:
                 download_urls(urls=[image_url],

From 5d7df2eb1bd1a8bd572e9ad656696870d0f297d4 Mon Sep 17 00:00:00 2001
From: Yang Bo <rslovers@yandex.com>
Date: Sun, 16 Dec 2018 07:23:59 +0000
Subject: [PATCH 0749/1225] Fix zhibo.tv regular expression.

---
 src/you_get/extractors/zhibo.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/zhibo.py b/src/you_get/extractors/zhibo.py
index 4aaa293e7d..1d2eadea79 100644
--- a/src/you_get/extractors/zhibo.py
+++ b/src/you_get/extractors/zhibo.py
@@ -37,11 +37,14 @@ def zhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     if is_live is not "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % is_live)
 
-    ourStreamName = r1(r"window.ourStreamName=\'([s\S'\s\.]*)\'\;[\s\S]*window.rtmpDefaultSource", html)
-    rtmpPollUrl = r1(r"window.rtmpPollUrl=\'([s\S'\s\.]*)\'\;[\s\S]*window.hlsDefaultSource", html)
-
-    #real_url = 'rtmp://220.194.213.56/live.zhibo.tv/8live/' + ourStreamName
-    real_url = rtmpPollUrl + ourStreamName
+    match = re.search(r"""
+    ourStreamName .*?
+    '(.*?)' .*?
+    rtmpHighSource .*?
+    '(.*?)' .*?
+    '(.*?)'
+    """, html, re.S | re.X)
+    real_url = match.group(3) + match.group(1) + match.group(2)
 
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:

From afb2db7c3c943fcc327f7bff254bece4ae5717f8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 17 Dec 2018 12:07:56 +0100
Subject: [PATCH 0750/1225] version 0.4.1193

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e0a79a8e5a..e89eb41a33 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1181'
+__version__ = '0.4.1193'

From fef2298b956219a2856632199e604e380da486f0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 18 Dec 2018 15:53:56 +0100
Subject: [PATCH 0751/1225] [instagram] they're forming in straight line

---
 src/you_get/extractors/instagram.py | 22 ++++++----------------
 1 file changed, 6 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 9dd7207d63..567e0dd7d7 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -29,14 +29,9 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 image_url = edge['node']['display_url']
                 if 'video_url' in edge['node']:
                     image_url = edge['node']['video_url']
-                try:
-                    image_url = image_url.split('?')[0]
-                    ext = image_url.split('.')[-1]
-                    size = int(get_head(image_url)['Content-Length'])
-                except:
-                    image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
-                    ext = image_url.split('.')[-1]
-                    size = int(get_head(image_url)['Content-Length'])
+                ext = image_url.split('?')[0].split('.')[-1]
+                size = int(get_head(image_url)['Content-Length'])
+
                 print_info(site_info, title, ext, size)
                 if not info_only:
                     download_urls(urls=[image_url],
@@ -49,14 +44,9 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
             if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
                 image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
-            try:
-                image_url = image_url.split('?')[0]
-                ext = image_url.split('.')[-1]
-                size = int(get_head(image_url)['Content-Length'])
-            except:
-                image_url = image_url.split('?')[0] + '?_nc_ht=instagram.fcph1-1.fna.fbcdn.net'
-                ext = image_url.split('.')[-1]
-                size = int(get_head(image_url)['Content-Length'])
+            ext = image_url.split('?')[0].split('.')[-1]
+            size = int(get_head(image_url)['Content-Length'])
+
             print_info(site_info, title, ext, size)
             if not info_only:
                 download_urls(urls=[image_url],

From 98fedfb2a277a2c4e77fc85adc3865025bc696f2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 18 Dec 2018 18:20:01 +0100
Subject: [PATCH 0752/1225] [miaopai] handle weibo.com/tv/v URLs

---
 src/you_get/extractors/miaopai.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index b827024b72..304ac17609 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -67,7 +67,10 @@ def miaopai_download_by_wbmp(wbmp_url, fid, info_only=False, **kwargs):
 
 def miaopai_download_direct(url, info_only, **kwargs):
     mobile_page = get_content(url, headers=fake_headers_mobile)
-    title = re.search(r'([\'"])title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    try:
+        title = re.search(r'([\'"])title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    except:
+        title = re.search(r'([\'"])status_title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
     title = title.replace('\n', '_')
     stream_url = re.search(r'([\'"])stream_url\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
     ext = 'mp4'
@@ -78,6 +81,9 @@ def miaopai_download_direct(url, info_only, **kwargs):
 
 # ----------------------------------------------------------------------
 def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
+    if match1(url, r'weibo\.com/tv/v/(\w+)'):
+        return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)

From b8470667568d723265ae1414b07be7c8cfdaa947 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 21 Dec 2018 15:46:11 +0100
Subject: [PATCH 0753/1225] [naver] fix #2671

---
 src/you_get/extractors/naver.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
index b9eef8d72e..add884e9bf 100644
--- a/src/you_get/extractors/naver.py
+++ b/src/you_get/extractors/naver.py
@@ -16,10 +16,15 @@ def naver_download_by_url(url, output_dir='.', merge=True, info_only=False, **kw
     ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'
     page = get_content(url)
     try:
-        og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page).group(1)
-        params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
-        vid = params_dict['vid'][0]
-        key = params_dict['outKey'][0]
+        temp = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page)
+        if temp is not None:
+            og_video_url = temp.group(1)
+            params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
+            vid = params_dict['vid'][0]
+            key = params_dict['outKey'][0]
+        else:
+            vid = re.search(r"\"videoId\"\s*:\s*\"(.+?)\"", page).group(1)
+            key = re.search(r"\"inKey\"\s*:\s*\"(.+?)\"", page).group(1)
         meta_str = get_content(ep.format(vid, key))
         meta_json = json.loads(meta_str)
         if 'errorCode' in meta_json:

From 4e98f7bcae333ad974a940bbd8fdb540cc9e1e9e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 22 Dec 2018 23:14:30 +0100
Subject: [PATCH 0754/1225] [tumblr] always download the high res (1280)
 version of images

---
 src/you_get/extractors/tumblr.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index d63aee728d..91b348fce3 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -49,17 +49,18 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         tuggles = {}
         for url in urls:
-            filename = parse.unquote(url.split('/')[-1])
+            hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg'  # FIXME: decide actual quality
+            filename = parse.unquote(hd_url.split('/')[-1])
             title = '.'.join(filename.split('.')[:-1])
             tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
             quality = int(r1(r'^tumblr_.+_(\d+)$', title))
             ext = filename.split('.')[-1]
             try:
-                size = int(get_head(url)['Content-Length'])
+                size = int(get_head(hd_url)['Content-Length'])
                 if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:
                     tuggles[tumblr_id] = {
                         'title': title,
-                        'url': url,
+                        'url': hd_url,
                         'quality': quality,
                         'ext': ext,
                         'size': size,

From 672cc4b1f90f355556b9fe800251382a41fb4b48 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 Dec 2018 18:21:28 +0100
Subject: [PATCH 0755/1225] [youtube] show warning for premieres

---
 src/you_get/extractors/youtube.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 9f2d286353..df2e9e4215 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -225,7 +225,11 @@ def prepare(self, **kwargs):
 
         # YouTube Live
         if ytplayer_config and (ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1'):
-            hlsvp = ytplayer_config['args']['hlsvp']
+            if 'hlsvp' in ytplayer_config['args']:
+                hlsvp = ytplayer_config['args']['hlsvp']
+            else:
+                player_response= json.loads(ytplayer_config['args']['player_response'])
+                log.e('[Failed] %s' % player_response['playabilityStatus']['reason'], exit_code=1)
 
             if 'info_only' in kwargs and kwargs['info_only']:
                 return

From 1df62c39ffb2ba3ddf115750cd3aa5d37895a81a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 26 Dec 2018 16:48:32 +0100
Subject: [PATCH 0756/1225] [universal] strip query string for direct download

---
 src/you_get/extractors/universal.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index a1ab153624..756ce4c1ca 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -137,7 +137,8 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
     else:
         # direct download
-        filename = parse.unquote(url.split('/')[-1]) or parse.unquote(url.split('/')[-2])
+        url_trunk = url.split('?')[0]  # strip query string
+        filename = parse.unquote(url_trunk.split('/')[-1]) or parse.unquote(url_trunk.split('/')[-2])
         title = '.'.join(filename.split('.')[:-1]) or filename
         _, ext, size = url_info(url, faker=True)
         print_info(site_info, title, ext, size)

From 0930bb1a0c89925c3fe22b6fb9635eda0f698dab Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 26 Dec 2018 16:57:25 +0100
Subject: [PATCH 0757/1225] update LICENSE.txt (making the MIT license
 detectable)

---
 LICENSE.txt | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index 7b25d906af..5964bf2038 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,15 +1,14 @@
-==============================================
-This is a copy of the MIT license.
-==============================================
-Copyright (C) 2012-2017 Mort Yao <mort.yao@gmail.com>
-Copyright (C) 2012 Boyu Guo <iambus@gmail.com>
+MIT License
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
+Copyright (c) 2012-2019 Mort Yao <mort.yao@gmail.com>
+Copyright (c) 2012 Boyu Guo <iambus@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
 The above copyright notice and this permission notice shall be included in all
 copies or substantial portions of the Software.

From 60e035cca8c0315a63621f7e6c5ab5d6fc260e8e Mon Sep 17 00:00:00 2001
From: trymelz <trymelz@yahoo.com>
Date: Tue, 1 Jan 2019 18:29:57 -0600
Subject: [PATCH 0758/1225] check if the player exist or not

---
 src/you_get/common.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7818216303..ae42e46b3c 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -273,7 +273,12 @@ def matchall(text, patterns):
 def launch_player(player, urls):
     import subprocess
     import shlex
-    subprocess.call(shlex.split(player) + list(urls))
+    import shutil
+    exefile=shlex.split(player)[0]
+    if shutil.which(exefile) is not None:
+        subprocess.call(shlex.split(player) + list(urls))
+    else:
+        log.wtf('[Failed] Cannot find player "%s"' % exefile)
 
 
 def parse_query_param(url, param):

From f09d7c958af2c98dc90e81b4a2b43717f39d860e Mon Sep 17 00:00:00 2001
From: trymelz <trymelz@yahoo.com>
Date: Tue, 1 Jan 2019 18:47:51 -0600
Subject: [PATCH 0759/1225] check python version before using shutil.which
 function

---
 src/you_get/common.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index ae42e46b3c..6d5764ff57 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -273,12 +273,15 @@ def matchall(text, patterns):
 def launch_player(player, urls):
     import subprocess
     import shlex
-    import shutil
-    exefile=shlex.split(player)[0]
-    if shutil.which(exefile) is not None:
-        subprocess.call(shlex.split(player) + list(urls))
+    if (sys.version_info >= (3, 3)):
+        import shutil
+        exefile=shlex.split(player)[0]
+        if shutil.which(exefile) is not None:
+            subprocess.call(shlex.split(player) + list(urls))
+        else:
+            log.wtf('[Failed] Cannot find player "%s"' % exefile)
     else:
-        log.wtf('[Failed] Cannot find player "%s"' % exefile)
+        subprocess.call(shlex.split(player) + list(urls))
 
 
 def parse_query_param(url, param):

From 95b711dc99dfc9958a6951a9dc7749ee8d58852c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Jan 2019 11:40:14 +0100
Subject: [PATCH 0760/1225] [tiktok] shake it

---
 src/you_get/extractors/tiktok.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index e9ff63abd0..9ecc662d78 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -5,7 +5,7 @@
 from ..common import *
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html(url)
+    html = get_html(url, faker=True)
     title = r1(r'<title>(.*?)</title>', html)
     video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
     title = '%s [%s]' % (title, video_id)

From 912804b0fa96f32c07635c13de0315f4e03aa643 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Jan 2019 12:09:43 +0100
Subject: [PATCH 0761/1225] [youtube] fix #2675

---
 src/you_get/extractors/youtube.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index df2e9e4215..9c05e7874e 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -41,6 +41,9 @@ def decipher(js, s):
         # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
         # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
         # - https://www.youtube.com/yts/jsbin/player-vfls4aurX/da_DK/base.js
+        # - https://www.youtube.com/yts/jsbin/player_ias-vfl_RGK2l/en_US/base.js
+        # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
+        # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             code = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', code)
@@ -56,7 +59,8 @@ def tr_js(code):
             return code
 
         js = js.replace('\n', ' ')
-        f1 = match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
+        f1 = match1(js, r'\.set\(\w+\.sp,encodeURIComponent\(([$\w]+)') or \
+            match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
             match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
             match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \

From a151a3e250af9116d92f9d4f2288a74ac0d7d4c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Jan 2019 12:26:57 +0100
Subject: [PATCH 0762/1225] version 0.4.1205

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e89eb41a33..b350385bc4 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1193'
+__version__ = '0.4.1205'

From 0aafca6de41e35b858eac56002829f56f352f328 Mon Sep 17 00:00:00 2001
From: wangqr <wangqr@wangqr.tk>
Date: Mon, 21 Jan 2019 02:11:41 -0500
Subject: [PATCH 0763/1225] Use copy when merging audio and video

---
 src/you_get/processor/ffmpeg.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index a18188dac9..51cc51dda5 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -59,12 +59,7 @@ def ffmpeg_concat_av(files, output, ext):
     params = [FFMPEG] + LOGLEVEL
     for file in files:
         if os.path.isfile(file): params.extend(['-i', file])
-    params.extend(['-c:v', 'copy'])
-    if ext == 'mp4':
-        params.extend(['-c:a', 'aac'])
-    elif ext == 'webm':
-        params.extend(['-c:a', 'vorbis'])
-    params.extend(['-strict', 'experimental'])
+    params.extend(['-c', 'copy'])
     params.append(output)
     return subprocess.call(params, stdin=STDIN)
 

From f07e909a72412bbb189269d3aa53c8721f9571b7 Mon Sep 17 00:00:00 2001
From: wangqr <wangqr@wangqr.tk>
Date: Mon, 21 Jan 2019 09:50:47 -0500
Subject: [PATCH 0764/1225] Fallback to re-encoding audio when ffmpeg quits
 with error

---
 src/you_get/processor/ffmpeg.py | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 51cc51dda5..8bdf77d290 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import logging
-import os.path
+import os
 import subprocess
 import sys
 from ..util.strings import parameterize
@@ -61,7 +61,23 @@ def ffmpeg_concat_av(files, output, ext):
         if os.path.isfile(file): params.extend(['-i', file])
     params.extend(['-c', 'copy'])
     params.append(output)
-    return subprocess.call(params, stdin=STDIN)
+    if subprocess.call(params, stdin=STDIN):
+        print('Merging without re-encode failed.\nTry again re-encoding audio... ', end="", flush=True)
+        try: os.remove(output)
+        except FileNotFoundError: pass
+        params = [FFMPEG] + LOGLEVEL
+        for file in files:
+            if os.path.isfile(file): params.extend(['-i', file])
+        params.extend(['-c:v', 'copy'])
+        if ext == 'mp4':
+            params.extend(['-c:a', 'aac'])
+            params.extend(['-strict', 'experimental'])
+        elif ext == 'webm':
+            params.extend(['-c:a', 'opus'])
+        params.append(output)
+        return subprocess.call(params, stdin=STDIN)
+    else:
+        return 0
 
 def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
     for file in files:

From a449eca64e9999b359ce50b38ceafab6710e94e8 Mon Sep 17 00:00:00 2001
From: sheerluck <sheer.luck.andrew@gmail.com>
Date: Mon, 4 Feb 2019 11:18:06 +0300
Subject: [PATCH 0765/1225] fix coub ffmpeg

Test URL:  https://coub.com/view/19cyub

1) ```params.extend(['-safe', '0'])``` fixes ```[concat @ 0x55b4c8f02980] Unsafe file name 'Test_For_The_Princess!.mp4'```
Before patch: Test_For_The_Princess!_full.mp4 is not created
After patch: Test_For_The_Princess!_full.mp4 created successfully

2) ```round``` fixes ```audio_duration 64.0 + video_duration 8.04 -> 7```
Before patch: video is freezed for last 8 second
After patch: perfect loop
---
 src/you_get/extractors/coub.py  | 4 ++--
 src/you_get/processor/ffmpeg.py | 1 +
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/coub.py b/src/you_get/extractors/coub.py
index 44e403d320..36a0a5d678 100644
--- a/src/you_get/extractors/coub.py
+++ b/src/you_get/extractors/coub.py
@@ -25,10 +25,10 @@ def coub_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
                 loop_file_path = get_loop_file_path(title, output_dir)
                 single_file_path = audio_file_path
                 if audio_duration > video_duration:
-                    write_loop_file(int(audio_duration / video_duration), loop_file_path, video_file_name)
+                    write_loop_file(round(audio_duration / video_duration), loop_file_path, video_file_name)
                 else:
                     single_file_path = audio_file_path
-                    write_loop_file(int(video_duration / audio_duration), loop_file_path, audio_file_name)
+                    write_loop_file(round(video_duration / audio_duration), loop_file_path, audio_file_name)
 
                 ffmpeg.ffmpeg_concat_audio_and_video([loop_file_path, single_file_path], title + "_full", "mp4")
                 cleanup_files([video_file_path, audio_file_path, loop_file_path])
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 8bdf77d290..02ecb012c2 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -267,6 +267,7 @@ def ffmpeg_concat_audio_and_video(files, output, ext):
     if has_ffmpeg_installed:
         params = [FFMPEG] + LOGLEVEL
         params.extend(['-f', 'concat'])
+        params.extend(['-safe', '0'])  # https://stackoverflow.com/questions/38996925/ffmpeg-concat-unsafe-file-name
         for file in files:
             if os.path.isfile(file):
                 params.extend(['-i', file])

From 4f191986bd6b9e9c73c437c38a1aec70cbaba57a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 10 Feb 2019 23:13:28 +0100
Subject: [PATCH 0766/1225] [miaopai] it is clear that

---
 src/you_get/extractors/miaopai.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 304ac17609..01d043f268 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -84,14 +84,15 @@ def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **
     if match1(url, r'weibo\.com/tv/v/(\w+)'):
         return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
+    if re.match(r'^http[s]://.*\.weibo\.com/\d+/.+', url):
+        return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)
     elif '/p/230444' in url:
         fid = match1(url, r'/p/230444(\w+)')
         miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
-    elif re.match(r'^http[s]://weibo\.com/\d+/.+', url):
-        miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
     else:
         mobile_page = get_content(url, headers = fake_headers_mobile)
         hit = re.search(r'"page_url"\s*:\s*"([^"]+)"', mobile_page)

From fd08f7b639b3cc8c8aa74b9f80a8b680a7c40f47 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 16 Feb 2019 06:09:57 +0100
Subject: [PATCH 0767/1225] [bilibili] fix #2680

---
 src/you_get/extractors/bilibili.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5ed7f28d05..4685311813 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -224,7 +224,13 @@ def entry(self, **kwargs):
                 p = int(page.group(1))
             cid = re.search(r'"cid":(\d+),"page":%s' % p, self.page).group(1)
         if cid is not None:
-            self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs)
+            #self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs)
+            # FIXME: video qualities
+            playinfo_text = match1(self.page, r'__playinfo__=(.*?)<')
+            playinfo = json.loads(playinfo_text)
+            url0 = playinfo['data']['durl'][0]['url']
+            _, ext, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
+            self.streams['flv'] = {'url': url0, 'container': ext, 'size': size, 'src': [url0]}
         else:
             # flashvars?
             flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)
@@ -492,7 +498,8 @@ def bilibili_download_playlist_by_url(url, **kwargs):
         page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
         page_cnt = len(page_list)
         for no in range(1, page_cnt+1):
-            page_url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, no)
+            #page_url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, no)
+            page_url = 'http://www.bilibili.com/video/av{}/?p={}'.format(aid, no)
             subtitle = '#%s. %s'% (page_list[no-1]['page'], page_list[no-1]['pagename'])
             Bilibili().download_by_url(page_url, subtitle=subtitle, **kwargs)
 

From e6534c41d87a7a37499816c26aeb49a44f965505 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 16 Feb 2019 06:17:11 +0100
Subject: [PATCH 0768/1225] version 0.4.1210

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index b350385bc4..f2b279b1f3 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1205'
+__version__ = '0.4.1210'

From 9dc034c2cc23f8a68244924bde2aac3f7db5514f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 16 Feb 2019 23:08:44 +0100
Subject: [PATCH 0769/1225] [common] download_urls(): fix URL printing for DASH
 streams

---
 src/you_get/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6d5764ff57..9735a30f6a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -916,7 +916,10 @@ def download_urls(
         return
     if dry_run:
         print_user_agent(faker=faker)
-        print('Real URLs:\n%s' % '\n'.join(urls))
+        try:
+            print('Real URLs:\n%s' % '\n'.join(urls))
+        except:
+            print('Real URLs:\n%s' % '\n'.join([j for i in urls for j in i]))
         return
 
     if player:

From a01a68fcb1cf7ed457b35075b98c9dcc0026774d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 16 Feb 2019 23:14:59 +0100
Subject: [PATCH 0770/1225] [bilibili] rewrite the extractor

---
 src/you_get/extractors/bilibili.py | 575 +++++------------------------
 1 file changed, 93 insertions(+), 482 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 4685311813..6c70a574b6 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -1,510 +1,121 @@
 #!/usr/bin/env python
 
-__all__ = ['bilibili_download']
-
-import hashlib
-import re
-import time
-import json
-import http.cookiejar
-import urllib.request
-import urllib.parse
-from xml.dom.minidom import parseString
-
 from ..common import *
-from ..util.log import *
-from ..extractor import *
-
-from .qq import qq_download_by_vid
-from .sina import sina_download_by_vid
-from .tudou import tudou_download_by_id
-from .youku import youku_download_by_vid
+from ..extractor import VideoExtractor
 
 class Bilibili(VideoExtractor):
-    name = 'Bilibili'
-    live_api = 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid={}&quality=0&platform=web'
-    api_url = 'http://interface.bilibili.com/v2/playurl?'
-    bangumi_api_url = 'http://bangumi.bilibili.com/player/web_api/playurl?'
-    live_room_init_api_url = 'https://api.live.bilibili.com/room/v1/Room/room_init?id={}'
-    live_room_info_api_url = 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id={}'
+    name = "Bilibili"
 
-    #SEC1 = '1c15888dc316e05a15fdd0a02ed6584f'
-    SEC1 = '94aba54af9065f71de72f5508f1cd42e'
-    SEC2 = '9b288147e5474dd2aa67085f716c560d'
+    # Bilibili media encoding options, in descending quality order.
     stream_types = [
-        {'id': 'hdflv'},
-        {'id': 'flv720'},
-        {'id': 'flv'},
-        {'id': 'hdmp4'},
-        {'id': 'mp4'},
-        {'id': 'live'},
-        {'id': 'vc'}
+        {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
+        {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P'},
+        {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P60'},
+        {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P'},
+        {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '480p', 'desc': '清晰 480P'},  # default
+        {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
+         'container': 'MP4', 'video_resolution': '360p', 'desc': '流畅 360P'},
     ]
-    fmt2qlt = dict(hdflv=4, flv=3, hdmp4=2, mp4=1)
 
     @staticmethod
-    def bilibili_stream_type(urls):
-        url = urls[0]
-        if 'hd.flv' in url or '-80.flv' in url:
-            return 'hdflv', 'flv'
-        if '-64.flv' in url:
-            return 'flv720', 'flv'
-        if '.flv' in url:
-            return 'flv', 'flv'
-        if 'hd.mp4' in url or '-48.mp4' in url:
-            return 'hdmp4', 'mp4'
-        if '.mp4' in url:
-            return 'mp4', 'mp4'
-        raise Exception('Unknown stream type')
-
-    def api_req(self, cid, quality, bangumi, bangumi_movie=False, **kwargs):
-        ts = str(int(time.time()))
-        if not bangumi:
-            #params_str = 'cid={}&player=1&quality={}&ts={}'.format(cid, quality, ts)
-            params_str = 'appkey=84956560bc028eb7&cid={}&otype=xml&qn={}&quality={}&type='.format(cid, quality, quality)
-            chksum = hashlib.md5(bytes(params_str+self.SEC1, 'utf8')).hexdigest()
-            api_url = self.api_url + params_str + '&sign=' + chksum
-        else:
-            mod = 'movie' if bangumi_movie else 'bangumi'
-            params_str = 'cid={}&module={}&player=1&quality={}&ts={}'.format(cid, mod, quality, ts)
-            chksum = hashlib.md5(bytes(params_str+self.SEC2, 'utf8')).hexdigest()
-            api_url = self.bangumi_api_url + params_str + '&sign=' + chksum
-
-        xml_str = get_content(api_url, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
-        return xml_str
-
-    def parse_bili_xml(self, xml_str):
-        urls_list = []
-        total_size = 0
-        doc = parseString(xml_str.encode('utf8'))
-        durls = doc.getElementsByTagName('durl')
-        for durl in durls:
-            size = durl.getElementsByTagName('size')[0]
-            total_size += int(size.firstChild.nodeValue)
-            url = durl.getElementsByTagName('url')[0]
-            urls_list.append(url.firstChild.nodeValue)
-        stream_type, container = self.bilibili_stream_type(urls_list)
-        if stream_type not in self.streams:
-            self.streams[stream_type] = {}
-            self.streams[stream_type]['src'] = urls_list
-            self.streams[stream_type]['size'] = total_size
-            self.streams[stream_type]['container'] = container
-
-    def download_by_vid(self, cid, bangumi, **kwargs):
-        stream_id = kwargs.get('stream_id')
-        # guard here. if stream_id invalid, fallback as not stream_id
-        if stream_id and stream_id in self.fmt2qlt:
-            quality = stream_id
-        else:
-            quality = 'hdflv' if bangumi else 'flv'
-
-        info_only = kwargs.get('info_only')
-        for qlt in [116,112,80,74,64,32,16,15]:
-            api_xml = self.api_req(cid, qlt, bangumi, **kwargs)
-            self.parse_bili_xml(api_xml)
-        if not info_only or stream_id:
-            self.danmuku = get_danmuku_xml(cid)
+    def bilibili_headers(referer=None, cookie=None):
+        # a reasonable UA
+        ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
+        headers = {'User-Agent': ua}
+        if referer is not None:
+            headers.update({'Referer': referer})
+        if cookie is not None:
+            headers.update({'Cookie': cookie})
+        return headers
 
     def prepare(self, **kwargs):
-        if socket.getdefaulttimeout() == 600: # no timeout specified
-            socket.setdefaulttimeout(2) # fail fast, very speedy!
-
-        # handle "watchlater" URLs
-        if '/watchlater/' in self.url:
-            aid = re.search(r'av(\d+)', self.url).group(1)
-            self.url = 'http://www.bilibili.com/video/av{}/'.format(aid)
+        self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
-        self.ua = fake_headers['User-Agent']
-        self.url = url_locations([self.url], faker=True)[0]
-        frag = urllib.parse.urlparse(self.url).fragment
-        # http://www.bilibili.com/video/av3141144/index_2.html#page=3
-        if frag:
-            hit = re.search(r'page=(\d+)', frag)
-            if hit is not None:
-                page = hit.group(1)
-                aid = re.search(r'av(\d+)', self.url).group(1)
-                self.url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, page)
-        self.referer = self.url
-        self.page = get_content(self.url, headers=fake_headers)
+        html_content = get_content(self.url, headers=self.bilibili_headers())
+        #self.title = match1(html_content,
+        #                    r'<h1 title="([^"]+)"')
 
-        m = re.search(r'<h1.*?>(.*?)</h1>', self.page) or re.search(r'<h1 title="([^"]+)">', self.page)
-        if m is not None:
-            self.title = m.group(1)
-            s = re.search(r'<span.*?>([^<]+)</span>', m.group(1))
-            if s:
-                self.title = unescape_html(s.group(1))
-        if self.title is None:
-            m = re.search(r'property="og:title" content="([^"]+)"', self.page)
-            if m is not None:
-                self.title = m.group(1)
-        if 'subtitle' in kwargs:
-            subtitle = kwargs['subtitle']
-            self.title = '{} {}'.format(self.title, subtitle)
-        else:
-            playinfo = re.search(r'__INITIAL_STATE__=(.*?);\(function\(\)', self.page)
-            if playinfo is not None:
-                jsonPlayinfo = json.loads(playinfo.group(1))
-                if 'videoData' in jsonPlayinfo:
-                    pages = jsonPlayinfo['videoData']['pages']
-                    if len(pages) > 1:
-                        qs = dict(parse.parse_qsl(urllib.parse.urlparse(self.url).query))
-                        page = pages[int(qs.get('p', 1)) - 1]
-                        self.title = '{} #{}. {}'.format(self.title, page['page'], page['part'])
-
-        if 'bangumi.bilibili.com/movie' in self.url:
-            self.movie_entry(**kwargs)
-        elif 'bangumi.bilibili.com' in self.url:
-            self.bangumi_entry(**kwargs)
-        elif 'bangumi/' in self.url:
-            self.bangumi_entry(**kwargs)
-        elif 'live.bilibili.com' in self.url:
-            self.live_entry(**kwargs)
-        elif 'vc.bilibili.com' in self.url:
-            self.vc_entry(**kwargs)
-        elif 'audio/au' in self.url:
-            self.audio_entry(**kwargs)
-        else:
-            self.entry(**kwargs)
-
-    def movie_entry(self, **kwargs):
-        patt = r"var\s*aid\s*=\s*'(\d+)'"
-        aid = re.search(patt, self.page).group(1)
-        page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
-        # better ideas for bangumi_movie titles?
-        self.title = page_list[0]['pagename']
-        self.download_by_vid(page_list[0]['cid'], True, bangumi_movie=True, **kwargs)
-
-    def audio_entry(self, **kwargs):
-        assert re.match(r'https?://www.bilibili.com/audio/au\d+', self.url)
-        patt = r"(\d+)"
-        audio_id = re.search(patt, self.url).group(1)
-        audio_info_url = \
-            'https://www.bilibili.com/audio/music-service-c/web/song/info?sid={}'.format(audio_id)
-        audio_info_response = json.loads(get_content(audio_info_url))
-        if audio_info_response['msg'] != 'success':
-            log.wtf('fetch audio information failed!')
-            sys.exit(2)
-        self.title = audio_info_response['data']['title']
-        # TODO:there is no quality option for now
-        audio_download_url = \
-            'https://www.bilibili.com/audio/music-service-c/web/url?sid={}&privilege=2&quality=2'.format(audio_id)
-        audio_download_response = json.loads(get_content(audio_download_url))
-        if audio_download_response['msg'] != 'success':
-            log.wtf('fetch audio resource failed!')
-            sys.exit(2)
-        self.streams['mp4'] = {}
-        self.streams['mp4']['src'] = [audio_download_response['data']['cdns'][0]]
-        self.streams['mp4']['container'] = 'm4a'
-        self.streams['mp4']['size'] = audio_download_response['data']['size']
-
-
-    def entry(self, **kwargs):
-        # tencent player
-        tc_flashvars = re.search(r'"bili-cid=\d+&bili-aid=\d+&vid=([^"]+)"', self.page)
-        if tc_flashvars:
-            tc_flashvars = tc_flashvars.group(1)
-        if tc_flashvars is not None:
-            self.out = True
-            qq_download_by_vid(tc_flashvars, self.title, True, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
-            return
-
-        has_plist = re.search(r'"page":2', self.page)
-        if has_plist and not kwargs.get('playlist'):
-            log.w('This page contains a playlist. (use --playlist to download all videos.)')
-
-        try:
-            page_list = json.loads(re.search(r'"pages":(\[.*?\])', self.page).group(1))
-            index_id = int(re.search(r'index_(\d+)', self.url).group(1))
-            cid = page_list[index_id-1]['cid'] # change cid match rule
-        except:
-            page = re.search(r'p=(\d+)', self.url)
-            if page is None:
-                p = 1
-            else:
-                p = int(page.group(1))
-            cid = re.search(r'"cid":(\d+),"page":%s' % p, self.page).group(1)
-        if cid is not None:
-            #self.download_by_vid(cid, re.search('bangumi', self.url) is not None, **kwargs)
-            # FIXME: video qualities
-            playinfo_text = match1(self.page, r'__playinfo__=(.*?)<')
+        # regular av
+        # TODO: multi-P
+        if re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+            playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text)
-            url0 = playinfo['data']['durl'][0]['url']
-            _, ext, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
-            self.streams['flv'] = {'url': url0, 'container': ext, 'size': size, 'src': [url0]}
-        else:
-            # flashvars?
-            flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)
-            if flashvars is None:
-                raise Exception('Unsupported page {}'.format(self.url))
-            param = flashvars.split('&')[0]
-            t, cid = param.split('=')
-            t = t.strip()
-            cid = cid.strip()
-            if t == 'vid':
-                sina_download_by_vid(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
-            elif t == 'ykid':
-                youku_download_by_vid(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
-            elif t == 'uid':
-                tudou_download_by_id(cid, self.title, output_dir=kwargs['output_dir'], merge=kwargs['merge'], info_only=kwargs['info_only'])
-            else:
-                raise NotImplementedError('Unknown flashvars {}'.format(flashvars))
-            return
 
-    def live_entry(self, **kwargs):
-        # Extract room ID from the short display ID (seen in the room
-        # URL). The room ID is usually the same as the short ID, but not
-        # always; case in point: https://live.bilibili.com/48, with 48
-        # as the short ID and 63727 as the actual ID.
-        room_short_id = re.search(r'live.bilibili.com/([^?]+)', self.url).group(1)
-        room_init_api_response = json.loads(get_content(self.live_room_init_api_url.format(room_short_id)))
-        self.room_id = room_init_api_response['data']['room_id']
-
-        room_info_api_response = json.loads(get_content(self.live_room_info_api_url.format(self.room_id)))
-        self.title = room_info_api_response['data']['title']
-
-        api_url = self.live_api.format(self.room_id)
-        json_data = json.loads(get_content(api_url))
-        urls = [json_data['data']['durl'][0]['url']]
-
-        self.streams['live'] = {}
-        self.streams['live']['src'] = urls
-        self.streams['live']['container'] = 'flv'
-        self.streams['live']['size'] = 0
-
-    def vc_entry(self, **kwargs):
-        vc_id = re.search(r'video/(\d+)', self.url)
-        if not vc_id:
-            vc_id = re.search(r'vcdetail\?vc=(\d+)', self.url)
-            if not vc_id:
-                log.wtf('Unknown url pattern')
-        endpoint = 'http://api.vc.bilibili.com/clip/v1/video/detail?video_id={}&need_playurl=1'.format(vc_id.group(1))
-        vc_meta = json.loads(get_content(endpoint, headers=fake_headers))
-        if vc_meta['code'] != 0:
-            log.wtf('{}\n{}'.format(vc_meta['msg'], vc_meta['message']))
-        item = vc_meta['data']['item']
-        self.title = item['description']
-
-        self.streams['vc'] = {}
-        self.streams['vc']['src'] = [item['video_playurl']]
-        self.streams['vc']['container'] = 'mp4'
-        self.streams['vc']['size'] = int(item['video_size'])
-
-    def bangumi_entry(self, **kwargs):
-        data = json.loads(re.search(r'__INITIAL_STATE__=(.+);\(function', self.page).group(1))
-        cid = data['epInfo']['cid']
-        # index_title = data['epInfo']['index_title']
-        self.download_by_vid(cid, bangumi=True, **kwargs)
-
-
-def check_oversea():
-    url = 'https://interface.bilibili.com/player?id=cid:17778881'
-    xml_lines = get_content(url).split('\n')
-    for line in xml_lines:
-        key = line.split('>')[0][1:]
-        if key == 'country':
-            value = line.split('>')[1].split('<')[0]
-            if value != '中国':
-                return True
-            else:
-                return False
-    return False
-
-def check_sid():
-    if not cookies:
-        return False
-    for cookie in cookies:
-        if cookie.domain == '.bilibili.com' and cookie.name == 'sid':
-            return True
-    return False
-
-def fetch_sid(cid, aid):
-    url = 'http://interface.bilibili.com/player?id=cid:{}&aid={}'.format(cid, aid)
-    cookies = http.cookiejar.CookieJar()
-    req = urllib.request.Request(url)
-    res = urllib.request.urlopen(url)
-    cookies.extract_cookies(res, req)
-    for c in cookies:
-        if c.domain == '.bilibili.com' and c.name == 'sid':
-            return c.value
-    raise
-
-def collect_bangumi_epids(json_data):
-    eps = json_data['episodes'][::-1]
-    return [ep['episode_id'] for ep in eps]
-
-def get_bangumi_info(season_id):
-    BASE_URL = 'http://bangumi.bilibili.com/jsonp/seasoninfo/'
-    long_epoch = int(time.time() * 1000)
-    req_url = BASE_URL + season_id + '.ver?callback=seasonListCallback&jsonp=jsonp&_=' + str(long_epoch)
-    season_data = get_content(req_url)
-    season_data = season_data[len('seasonListCallback('):]
-    season_data = season_data[: -1 * len(');')]
-    json_data = json.loads(season_data)
-    return json_data['result']
-
-def get_danmuku_xml(cid):
-    return get_content('http://comment.bilibili.com/{}.xml'.format(cid))
-
-def parse_cid_playurl(xml):
-    from xml.dom.minidom import parseString
-    try:
-        urls_list = []
-        total_size = 0
-        doc = parseString(xml.encode('utf-8'))
-        durls = doc.getElementsByTagName('durl')
-        cdn_cnt = len(durls[0].getElementsByTagName('url'))
-        for i in range(cdn_cnt):
-            urls_list.append([])
-        for durl in durls:
-            size = durl.getElementsByTagName('size')[0]
-            total_size += int(size.firstChild.nodeValue)
-            cnt = len(durl.getElementsByTagName('url'))
-            for i in range(cnt):
-                u = durl.getElementsByTagName('url')[i].firstChild.nodeValue
-                urls_list[i].append(u)
-        return urls_list, total_size
-    except Exception as e:
-        log.w(e)
-        return [], 0
-
-def download_video_from_favlist(url, **kwargs):
-    # the url has format: https://space.bilibili.com/64169458/#/favlist?fid=1840028
-
-    m = re.search(r'space\.bilibili\.com/(\d+)/.*?fid=(\d+).*?', url)
-    vmid = ""
-    favid = ""
-    if m is not None:
-        vmid = m.group(1)
-        favid = m.group(2)
-        jsonresult = json.loads(get_content("https://api.bilibili.com/x/space/fav/arc?vmid={}&ps=300&fid={}&order=fav_time&tid=0&keyword=&pn=1&jsonp=jsonp".format(vmid, favid)))
-
-        # log.wtf("Got files list for vmid" + vmid + " favid:" + favid)
-        if jsonresult['code'] != 0:
-            log.wtf("Fail to get the files of page " + jsonresult)
-            sys.exit(2)
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            self.title = initial_state['videoData']['title']
+
+            # determine default quality / format
+            quality = int(playinfo['data']['quality'])
+            format_id = self.stream_qualities[quality]['id']
+            container = self.stream_qualities[quality]['container'].lower()
+            desc = self.stream_qualities[quality]['desc']
+            self.stream_types.append({'id': 'default'})
+
+            # determine default source URL and size
+            src, size = [], 0
+            for durl in playinfo['data']['durl']:
+                src.append(durl['url'])
+                size += durl['size']
+            self.streams['default'] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+            # DASH formats
+            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo_ = json.loads(playinfo_text_)
+            for video in playinfo_['data']['dash']['video']:
+                # prefer the latter codecs!
+                s = self.stream_qualities[video['id']]
+                format_id = s['id']
+                container = s['container'].lower()
+                desc = s['desc']
+                audio_quality = s['audio_quality']
+                baseurl = video['baseUrl']
+                size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                # find matching audio track
+                audio_baseurl = playinfo_['data']['dash']['audio'][0]['baseUrl']
+                for audio in playinfo_['data']['dash']['audio']:
+                    if int(audio['id']) == audio_quality:
+                        audio_baseurl = audio['baseUrl']
+                        break
+                size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         else:
-            videos = jsonresult['data']['archives']
-            videocount = len(videos)
-            for i in range(videocount):
-                videoid = videos[i]["aid"]
-                videotitle = videos[i]["title"]
-                videourl = "https://www.bilibili.com/video/av{}".format(videoid)
-                print("Start downloading ", videotitle, " video ", videotitle)
-                Bilibili().download_by_url(videourl, subtitle=videotitle, **kwargs)
+            # NOT IMPLEMENTED
+            pass
 
-    else:
-        log.wtf("Fail to parse the fav title" + url, "")
-def download_music_from_favlist(url, page, **kwargs):
-    m = re.search(r'https?://www.bilibili.com/audio/mycollection/(\d+)', url)
-    if m is not None:
-        sid = m.group(1)
-        json_result = json.loads(get_content("https://www.bilibili.com/audio/music-service-c/web/song/of-coll?"
-                                             "sid={}&pn={}&ps=100".format(sid, page)))
-        if json_result['msg'] == 'success':
-            music_list = json_result['data']['data']
-            music_count = len(music_list)
-            for i in range(music_count):
-                audio_id = music_list[i]['id']
-                audio_title = music_list[i]['title']
-                audio_url = "https://www.bilibili.com/audio/au{}".format(audio_id)
-                print("Start downloading music ", audio_title)
-                Bilibili().download_by_url(audio_url, **kwargs)
-            if page < json_result['data']['pageCount']:
-                page += 1
-                download_music_from_favlist(url, page, **kwargs)
-        else:
-            log.wtf("Fail to get music list of page " + json_result)
-            sys.exit(2)
-    else:
-        log.wtf("Fail to parse the sid from " + url, "")
-
-def download_video_from_totallist(url, page, **kwargs):
-    # the url has format: https://space.bilibili.com/64169458/#/video
-    m = re.search(r'space\.bilibili\.com/(\d+)/.*?video', url)
-    mid = ""
-    if m is not None:
-        mid = m.group(1)
-        jsonresult = json.loads(get_content("https://space.bilibili.com/ajax/member/getSubmitVideos?mid={}&pagesize=100&tid=0&page={}&keyword=&order=pubdate&jsonp=jsonp".format(mid, page)))
-        if jsonresult['status']:
-            videos = jsonresult['data']['vlist']
-            videocount = len(videos)
-            for i in range(videocount):
-                videoid = videos[i]["aid"]
-                videotitle = videos[i]["title"]
-                videourl = "https://www.bilibili.com/video/av{}".format(videoid)
-                print("Start downloading ", videotitle, " video ", videotitle)
-                Bilibili().download_by_url(videourl, subtitle=videotitle, **kwargs)
-            if page < jsonresult['data']['pages']:
-                page += 1
-                download_video_from_totallist(url, page, **kwargs)
-        else:
-            log.wtf("Fail to get the files of page " + jsonresult)
-            sys.exit(2)
+    def extract(self, **kwargs):
+        # set UA and referer for downloading
+        headers = self.bilibili_headers(referer=self.url)
+        self.ua, self.referer = headers['User-Agent'], headers['Referer']
 
-    else:
-        log.wtf("Fail to parse the video title" + url, "")
+        if not self.streams_sorted:
+            # no stream is available
+            return
 
-def download_music_from_totallist(url, page, **kwargs):
-    m = re.search(r'https?://www.bilibili.com/audio/am(\d+)\?type=\d', url)
-    if m is not None:
-        sid = m.group(1)
-        json_result = json.loads(get_content("https://www.bilibili.com/audio/music-service-c/web/song/of-menu?"
-                                             "sid={}&pn={}&ps=100".format(sid, page)))
-        if json_result['msg'] == 'success':
-            music_list = json_result['data']['data']
-            music_count = len(music_list)
-            for i in range(music_count):
-                audio_id = music_list[i]['id']
-                audio_title = music_list[i]['title']
-                audio_url = "https://www.bilibili.com/audio/au{}".format(audio_id)
-                print("Start downloading music ",audio_title)
-                Bilibili().download_by_url(audio_url, **kwargs)
-            if page < json_result['data']['pageCount']:
-                page += 1
-                download_music_from_totallist(url, page, **kwargs)
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # extract the stream
+            stream_id = kwargs['stream_id']
+            if stream_id not in self.streams and stream_id not in self.dash_streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
         else:
-            log.wtf("Fail to get music list of page " + json_result)
-            sys.exit(2)
-    else:
-        log.wtf("Fail to parse the sid from " + url, "")
-
-def bilibili_download_playlist_by_url(url, **kwargs):
-    url = url_locations([url], faker=True)[0]
-    kwargs['playlist'] = True
-    # a bangumi here? possible?
-    if 'live.bilibili' in url:
-        site.download_by_url(url)
-    elif 'bangumi.bilibili' in url:
-        bangumi_id = re.search(r'(\d+)', url).group(1)
-        bangumi_data = get_bangumi_info(bangumi_id)
-        ep_ids = collect_bangumi_epids(bangumi_data)
+            # extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
 
-        base_url = url.split('#')[0]
-        for ep_id in ep_ids:
-            ep_url = '#'.join([base_url, ep_id])
-            Bilibili().download_by_url(ep_url, **kwargs)
-    elif 'favlist' in url:
-        # this a fav list folder
-        download_video_from_favlist(url, **kwargs)
-    elif re.match(r'https?://space.bilibili.com/\d+/#/video', url):
-        download_video_from_totallist(url, 1, **kwargs)
-    elif re.match(r'https://www.bilibili.com/audio/mycollection/\d+', url):
-        download_music_from_favlist(url, 1, **kwargs)
-    elif re.match(r'https?://www.bilibili.com/audio/am\d+\?type=\d', url):
-        download_music_from_totallist(url, 1, **kwargs)
-    else:
-        aid = re.search(r'av(\d+)', url).group(1)
-        page_list = json.loads(get_content('http://www.bilibili.com/widget/getPageList?aid={}'.format(aid)))
-        page_cnt = len(page_list)
-        for no in range(1, page_cnt+1):
-            #page_url = 'http://www.bilibili.com/video/av{}/index_{}.html'.format(aid, no)
-            page_url = 'http://www.bilibili.com/video/av{}/?p={}'.format(aid, no)
-            subtitle = '#%s. %s'% (page_list[no-1]['page'], page_list[no-1]['pagename'])
-            Bilibili().download_by_url(page_url, subtitle=subtitle, **kwargs)
 
 site = Bilibili()
 download = site.download_by_url
-download_playlist = bilibili_download_playlist_by_url
+# TODO: download_playlist
 
 bilibili_download = download

From 10ec4723b7477293b4f566e92b47ca09878bdf68 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 00:55:29 +0100
Subject: [PATCH 0771/1225] [bilibili] support multipart videos

---
 src/you_get/extractors/bilibili.py | 35 ++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 6c70a574b6..eb036225c2 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -17,9 +17,11 @@ class Bilibili(VideoExtractor):
         {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
          'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P'},
         {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '480p', 'desc': '清晰 480P'},  # default
+         'container': 'MP4', 'video_resolution': '480p', 'desc': '清晰 480P'},
         {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
          'container': 'MP4', 'video_resolution': '360p', 'desc': '流畅 360P'},
+
+        {'id': 'default', 'quality': 0}
     ]
 
     @staticmethod
@@ -41,7 +43,6 @@ def prepare(self, **kwargs):
         #                    r'<h1 title="([^"]+)"')
 
         # regular av
-        # TODO: multi-P
         if re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text)
@@ -50,12 +51,22 @@ def prepare(self, **kwargs):
             initial_state = json.loads(initial_state_text)
             self.title = initial_state['videoData']['title']
 
+            # refine title for a specific part
+            p = match1(self.url, r'\?p=(\d+)')  # use URL to decide p-number, not initial_state['p']
+            if p is not None:
+                part = initial_state['videoData']['pages'][int(p) - 1]['part']
+                self.title = '%s (P%s. %s)' % (self.title, p, part)
+
+            # warn if it is a multi-part video
+            pn = initial_state['videoData']['videos']
+            if pn > 1 and not kwargs.get('playlist'):
+                log.w('This is a multipart video. (use --playlist to download all parts.)')
+
             # determine default quality / format
             quality = int(playinfo['data']['quality'])
             format_id = self.stream_qualities[quality]['id']
             container = self.stream_qualities[quality]['container'].lower()
             desc = self.stream_qualities[quality]['desc']
-            self.stream_types.append({'id': 'default'})
 
             # determine default source URL and size
             src, size = [], 0
@@ -113,9 +124,25 @@ def extract(self, **kwargs):
             # extract stream with the best quality
             stream_id = self.streams_sorted[0]['id']
 
+    def download_playlist_by_url(self, url, **kwargs):
+        self.url = url
+        kwargs['playlist'] = True
+
+        html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # regular av
+        if re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            aid = initial_state['videoData']['aid']
+            pn = initial_state['videoData']['videos']
+            for pi in range(1, pn + 1):
+                purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
+                self.__class__().download_by_url(purl, **kwargs)
+
 
 site = Bilibili()
 download = site.download_by_url
-# TODO: download_playlist
+download_playlist = site.download_playlist_by_url
 
 bilibili_download = download

From 09ca6396409f5ccfe5d873b520b528e31e47e512 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 02:34:11 +0100
Subject: [PATCH 0772/1225] [bilibili] support /bangumi eps

---
 src/you_get/extractors/bilibili.py | 47 ++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index eb036225c2..c35763dbb7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -10,6 +10,7 @@ class Bilibili(VideoExtractor):
     stream_types = [
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
          'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
+        # 'id': 'hdflv2', 'quality': 112?
         {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
          'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P'},
         {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
@@ -35,6 +36,10 @@ def bilibili_headers(referer=None, cookie=None):
             headers.update({'Cookie': cookie})
         return headers
 
+    @staticmethod
+    def bilibili_bangumi_api(avid, cid, ep_id):
+        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=0&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, ep_id)
+
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
@@ -100,6 +105,48 @@ def prepare(self, **kwargs):
                 self.dash_streams[format_id] = {'container': container, 'quality': desc,
                                                 'src': [[baseurl], [audio_baseurl]], 'size': size}
 
+        # bangumi
+        elif re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            self.title = initial_state['h1Title']
+
+            ep_id = initial_state['epInfo']['id']
+            avid = initial_state['epInfo']['aid']
+            cid = initial_state['epInfo']['cid']
+            api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            data = json.loads(api_content)
+            for video in data['result']['dash']['video']:
+                # convert height to quality code
+                if video['height'] == 360:
+                    quality = 16
+                elif video['height'] == 480:
+                    quality = 32
+                elif video['height'] == 720:
+                    quality = 64
+                elif video['height'] == 1080:
+                    quality = 80
+                s = self.stream_qualities[quality]
+                format_id = s['id']
+                container = s['container'].lower()
+                desc = s['desc']
+                audio_quality = s['audio_quality']
+                baseurl = video['baseUrl']
+                size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                # find matching audio track
+                audio_baseurl = data['result']['dash']['audio'][0]['baseUrl']
+                for audio in data['result']['dash']['audio']:
+                    if int(audio['id']) == audio_quality:
+                        audio_baseurl = audio['baseUrl']
+                        break
+                size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+
         else:
             # NOT IMPLEMENTED
             pass

From e8f0b78a93a37eab49fe02fd64184a9090a91b23 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 14:02:29 +0100
Subject: [PATCH 0773/1225] [extractor] print default streams only if available

---
 src/you_get/extractor.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 8aeed560f0..d7d1209595 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -154,9 +154,10 @@ def p(self, stream_id=None):
                 for stream in itags:
                     self.p_stream(stream)
             # Print all other available streams
-            print("    [ DEFAULT ] %s" % ('_' * 33))
-            for stream in self.streams_sorted:
-                self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
+            if self.streams_sorted:
+                print("    [ DEFAULT ] %s" % ('_' * 33))
+                for stream in self.streams_sorted:
+                    self.p_stream(stream['id'] if 'id' in stream else stream['itag'])
 
         if self.audiolang:
             print("audio-languages:")

From 1b42b0bbf8dc5bf3f4147a23ca858eb0c4fb5e9a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 14:40:36 +0100
Subject: [PATCH 0774/1225] [bilibili] support /bangumi ss

---
 src/you_get/extractors/bilibili.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c35763dbb7..6deb0caade 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -47,8 +47,23 @@ def prepare(self, **kwargs):
         #self.title = match1(html_content,
         #                    r'<h1 title="([^"]+)"')
 
-        # regular av
-        if re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+        # redirect: bangumi/play/ss -> bangumi/play/ep
+        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ss(\d+)', self.url):
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            ep_id = initial_state['epList'][0]['id']
+            self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+
+        # regular av video
+        if sort == 'video':
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text)
 
@@ -106,7 +121,7 @@ def prepare(self, **kwargs):
                                                 'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         # bangumi
-        elif re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+        elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             self.title = initial_state['h1Title']

From cea03c2b67140ce48245e2bddeb2f36f925be3d7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 15:47:53 +0100
Subject: [PATCH 0775/1225] [bilibili] support bangumi as a playlist

---
 src/you_get/extractors/bilibili.py | 40 +++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 6deb0caade..cad820734a 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -55,6 +55,7 @@ def prepare(self, **kwargs):
             self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
+        # sort it out
         if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
@@ -65,7 +66,7 @@ def prepare(self, **kwargs):
         # regular av video
         if sort == 'video':
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo = json.loads(playinfo_text)
+            playinfo = json.loads(playinfo_text) if playinfo_text else None
 
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
@@ -82,6 +83,14 @@ def prepare(self, **kwargs):
             if pn > 1 and not kwargs.get('playlist'):
                 log.w('This is a multipart video. (use --playlist to download all parts.)')
 
+            # no playinfo is found
+            if playinfo is None:
+                # use bilibili error video instead
+                url = 'https://static.hdslb.com/error.mp4'
+                _, container, size = url_info(url)
+                self.streams['default'] = {'container': container, 'size': size, 'src': [url]}
+                return
+
             # determine default quality / format
             quality = int(playinfo['data']['quality'])
             format_id = self.stream_qualities[quality]['id']
@@ -126,12 +135,21 @@ def prepare(self, **kwargs):
             initial_state = json.loads(initial_state_text)
             self.title = initial_state['h1Title']
 
+            # warn if this bangumi has more than 1 video
+            epn = len(initial_state['epList'])
+            if epn > 1 and not kwargs.get('playlist'):
+                log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
+
             ep_id = initial_state['epInfo']['id']
             avid = initial_state['epInfo']['aid']
             cid = initial_state['epInfo']['cid']
             api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
             api_content = get_content(api_url, headers=self.bilibili_headers())
             data = json.loads(api_content)
+            if data['code'] < 0:  # error
+                log.e(data['message'])
+                return
+
             for video in data['result']['dash']['video']:
                 # convert height to quality code
                 if video['height'] == 360:
@@ -192,8 +210,16 @@ def download_playlist_by_url(self, url, **kwargs):
 
         html_content = get_content(self.url, headers=self.bilibili_headers())
 
-        # regular av
-        if re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+        # sort it out
+        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+
+        # regular av video
+        if sort == 'video':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             aid = initial_state['videoData']['aid']
@@ -202,6 +228,14 @@ def download_playlist_by_url(self, url, **kwargs):
                 purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
                 self.__class__().download_by_url(purl, **kwargs)
 
+        elif sort == 'bangumi':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            for ep in initial_state['epList']:
+                ep_id = ep['id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+
 
 site = Bilibili()
 download = site.download_by_url

From 5544dedeccabd3233c7138287c9b1a0a4acf1773 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 17:20:15 +0100
Subject: [PATCH 0776/1225] [extractor] only if 'container' in stream

---
 src/you_get/extractor.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index d7d1209595..0eef47c118 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -105,7 +105,7 @@ def p_stream(self, stream_id):
         if 'quality' in stream:
             print("      quality:       %s" % stream['quality'])
 
-        if 'size' in stream and stream['container'].lower() != 'm3u8':
+        if 'size' in stream and 'container' in stream and stream['container'].lower() != 'm3u8':
             if stream['size'] != float('inf')  and stream['size'] != 0:
                 print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
 

From 21d7d029495f98a2d66edc2e6d022ec26f94239c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 17:21:27 +0100
Subject: [PATCH 0777/1225] [bilibili] fix for videos without DASH formats

---
 src/you_get/extractors/bilibili.py | 45 +++++++++++++++---------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index cad820734a..3469233f0c 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -37,8 +37,8 @@ def bilibili_headers(referer=None, cookie=None):
         return headers
 
     @staticmethod
-    def bilibili_bangumi_api(avid, cid, ep_id):
-        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=0&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, ep_id)
+    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
+        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
 
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
@@ -108,26 +108,27 @@ def prepare(self, **kwargs):
             html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
             playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo_ = json.loads(playinfo_text_)
-            for video in playinfo_['data']['dash']['video']:
-                # prefer the latter codecs!
-                s = self.stream_qualities[video['id']]
-                format_id = s['id']
-                container = s['container'].lower()
-                desc = s['desc']
-                audio_quality = s['audio_quality']
-                baseurl = video['baseUrl']
-                size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                # find matching audio track
-                audio_baseurl = playinfo_['data']['dash']['audio'][0]['baseUrl']
-                for audio in playinfo_['data']['dash']['audio']:
-                    if int(audio['id']) == audio_quality:
-                        audio_baseurl = audio['baseUrl']
-                        break
-                size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                'src': [[baseurl], [audio_baseurl]], 'size': size}
+            if 'dash' in playinfo_['data']:
+                for video in playinfo_['data']['dash']['video']:
+                    # prefer the latter codecs!
+                    s = self.stream_qualities[video['id']]
+                    format_id = s['id']
+                    container = s['container'].lower()
+                    desc = s['desc']
+                    audio_quality = s['audio_quality']
+                    baseurl = video['baseUrl']
+                    size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    # find matching audio track
+                    audio_baseurl = playinfo_['data']['dash']['audio'][0]['baseUrl']
+                    for audio in playinfo_['data']['dash']['audio']:
+                        if int(audio['id']) == audio_quality:
+                            audio_baseurl = audio['baseUrl']
+                            break
+                    size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         # bangumi
         elif sort == 'bangumi':

From 6e89b8a1e194f3e96451411c50d4d950b5a88814 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 17:31:27 +0100
Subject: [PATCH 0778/1225] [bilibili] assume container to be mp4 (#2681)

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 4685311813..dbab756341 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -229,8 +229,8 @@ def entry(self, **kwargs):
             playinfo_text = match1(self.page, r'__playinfo__=(.*?)<')
             playinfo = json.loads(playinfo_text)
             url0 = playinfo['data']['durl'][0]['url']
-            _, ext, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
-            self.streams['flv'] = {'url': url0, 'container': ext, 'size': size, 'src': [url0]}
+            _, _, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
+            self.streams['flv'] = {'url': url0, 'container': 'mp4', 'size': size, 'src': [url0]}
         else:
             # flashvars?
             flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)

From 451dfa6055b63a54d4a3b8e9a47801163a1735ad Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Feb 2019 23:04:18 +0100
Subject: [PATCH 0779/1225] [bilibili] refine titles correctly

---
 src/you_get/extractors/bilibili.py | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 3469233f0c..decb923252 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -36,6 +36,10 @@ def bilibili_headers(referer=None, cookie=None):
             headers.update({'Cookie': cookie})
         return headers
 
+    @staticmethod
+    def bilibili_api(avid, cid, qn=0):
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
+
     @staticmethod
     def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
         return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
@@ -70,19 +74,20 @@ def prepare(self, **kwargs):
 
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
-            self.title = initial_state['videoData']['title']
-
-            # refine title for a specific part
-            p = match1(self.url, r'\?p=(\d+)')  # use URL to decide p-number, not initial_state['p']
-            if p is not None:
-                part = initial_state['videoData']['pages'][int(p) - 1]['part']
-                self.title = '%s (P%s. %s)' % (self.title, p, part)
 
             # warn if it is a multi-part video
             pn = initial_state['videoData']['videos']
             if pn > 1 and not kwargs.get('playlist'):
                 log.w('This is a multipart video. (use --playlist to download all parts.)')
 
+            # set video title
+            self.title = initial_state['videoData']['title']
+            # refine title for a specific part, if it is a multi-part video
+            p = int(match1(self.url, r'[\?&]p=(\d+)') or '1')  # use URL to decide p-number, not initial_state['p']
+            if pn > 1:
+                part = initial_state['videoData']['pages'][p - 1]['part']
+                self.title = '%s (P%s. %s)' % (self.title, p, part)
+
             # no playinfo is found
             if playinfo is None:
                 # use bilibili error video instead
@@ -134,6 +139,8 @@ def prepare(self, **kwargs):
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
+
+            # set video title
             self.title = initial_state['h1Title']
 
             # warn if this bangumi has more than 1 video

From 56e4c5c9de4f1b69439c9c45e1195c7060f98d48 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 03:05:23 +0100
Subject: [PATCH 0780/1225] [bilibili] hard to explain but this fixes a lot

---
 src/you_get/extractors/bilibili.py | 127 +++++++++++++++++------------
 1 file changed, 75 insertions(+), 52 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index decb923252..04b09b19a8 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -9,20 +9,18 @@ class Bilibili(VideoExtractor):
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
         # 'id': 'hdflv2', 'quality': 112?
         {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '1080p', 'desc': '高清 1080P'},
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
         {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P60'},
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
         {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P'},
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
         {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '480p', 'desc': '清晰 480P'},
+         'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
         {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
-         'container': 'MP4', 'video_resolution': '360p', 'desc': '流畅 360P'},
-
-        {'id': 'default', 'quality': 0}
+         'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
     ]
 
     @staticmethod
@@ -38,7 +36,7 @@ def bilibili_headers(referer=None, cookie=None):
 
     @staticmethod
     def bilibili_api(avid, cid, qn=0):
-        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
 
     @staticmethod
     def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
@@ -69,11 +67,15 @@ def prepare(self, **kwargs):
 
         # regular av video
         if sort == 'video':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text) if playinfo_text else None
 
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
+            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
 
             # warn if it is a multi-part video
             pn = initial_state['videoData']['videos']
@@ -88,52 +90,73 @@ def prepare(self, **kwargs):
                 part = initial_state['videoData']['pages'][p - 1]['part']
                 self.title = '%s (P%s. %s)' % (self.title, p, part)
 
-            # no playinfo is found
-            if playinfo is None:
+            # construct playinfos
+            avid = initial_state['aid']
+            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
+
+            quality = 80  # best expectable quality
+            if playinfo is not None:
+                quality = playinfo['data']['quality'] or quality  # 0 indicates an error, fallback to best
+            playinfos = []
+            if playinfo is not None:
+                playinfos.append(playinfo)
+            if playinfo_ is not None:
+                playinfos.append(playinfo_)
+            # get lower formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # For dash, qn does not matter
+                if qn < quality:
+                    api_url = self.bilibili_api(avid, cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
+                    else:
+                        message = api_playinfo['data']['message']
+            if not playinfos:
+                log.w(message)
                 # use bilibili error video instead
                 url = 'https://static.hdslb.com/error.mp4'
                 _, container, size = url_info(url)
-                self.streams['default'] = {'container': container, 'size': size, 'src': [url]}
+                self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
                 return
 
-            # determine default quality / format
-            quality = int(playinfo['data']['quality'])
-            format_id = self.stream_qualities[quality]['id']
-            container = self.stream_qualities[quality]['container'].lower()
-            desc = self.stream_qualities[quality]['desc']
-
-            # determine default source URL and size
-            src, size = [], 0
-            for durl in playinfo['data']['durl']:
-                src.append(durl['url'])
-                size += durl['size']
-            self.streams['default'] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-            # DASH formats
-            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo_ = json.loads(playinfo_text_)
-            if 'dash' in playinfo_['data']:
-                for video in playinfo_['data']['dash']['video']:
-                    # prefer the latter codecs!
-                    s = self.stream_qualities[video['id']]
-                    format_id = s['id']
-                    container = s['container'].lower()
-                    desc = s['desc']
-                    audio_quality = s['audio_quality']
-                    baseurl = video['baseUrl']
-                    size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                    # find matching audio track
-                    audio_baseurl = playinfo_['data']['dash']['audio'][0]['baseUrl']
-                    for audio in playinfo_['data']['dash']['audio']:
-                        if int(audio['id']) == audio_quality:
-                            audio_baseurl = audio['baseUrl']
-                            break
-                    size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
+            for playinfo in playinfos:
+                quality = playinfo['data']['quality']
+                format_id = self.stream_qualities[quality]['id']
+                container = self.stream_qualities[quality]['container'].lower()
+                desc = self.stream_qualities[quality]['desc']
+
+                if 'durl' in playinfo['data']:
+                    src, size = [], 0
+                    for durl in playinfo['data']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['data']:
+                    for video in playinfo['data']['dash']['video']:
+                        # prefer the latter codecs!
+                        s = self.stream_qualities[video['id']]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['data']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         # bangumi
         elif sort == 'bangumi':

From 416757454296bb323e64f82c7d5a1dbbd6685f9e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 03:27:46 +0100
Subject: [PATCH 0781/1225] [extractor] use best quality from dash_streams if
 streams_sorted is empty

---
 src/you_get/extractor.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 0eef47c118..996d9a0633 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -196,7 +196,10 @@ def download(self, **kwargs):
             else:
                 # Download stream with the best quality
                 from .processor.ffmpeg import has_ffmpeg_installed
-                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                if self.streams_sorted:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                else:
+                    stream_id = list(self.dash_streams)[-1]
 
             if 'index' not in kwargs:
                 self.p(stream_id)

From a46df576662b35c6b4a1d0e2676cbd24f9b64dd9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 03:28:35 +0100
Subject: [PATCH 0782/1225] [bilibili] fix DASH formats for bangumi

---
 src/you_get/extractors/bilibili.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 04b09b19a8..1719150c8d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -181,6 +181,7 @@ def prepare(self, **kwargs):
                 log.e(data['message'])
                 return
 
+            # DASH formats
             for video in data['result']['dash']['video']:
                 # convert height to quality code
                 if video['height'] == 360:
@@ -192,8 +193,8 @@ def prepare(self, **kwargs):
                 elif video['height'] == 1080:
                     quality = 80
                 s = self.stream_qualities[quality]
-                format_id = s['id']
-                container = s['container'].lower()
+                format_id = 'dash-' + s['id']  # prefix
+                container = 'mp4'  # enforce MP4 container
                 desc = s['desc']
                 audio_quality = s['audio_quality']
                 baseurl = video['baseUrl']

From 45045ccc3dee4c0d16fd75cbca0f2676472cf218 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 15:09:34 +0100
Subject: [PATCH 0783/1225] [bilibili] call interface v2 API (for better
 qualities without login)

---
 src/you_get/extractors/bilibili.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1719150c8d..22a50e7b6f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -3,6 +3,8 @@
 from ..common import *
 from ..extractor import VideoExtractor
 
+import hashlib
+
 class Bilibili(VideoExtractor):
     name = "Bilibili"
 
@@ -42,6 +44,14 @@ def bilibili_api(avid, cid, qn=0):
     def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
         return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
 
+    @staticmethod
+    def bilibili_interface_api(cid, qn=0):
+        entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
+        appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
+        params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
+        chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
+        return 'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
@@ -93,20 +103,21 @@ def prepare(self, **kwargs):
             # construct playinfos
             avid = initial_state['aid']
             cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
-
-            quality = 80  # best expectable quality
+            current_quality, best_quality = None, None
             if playinfo is not None:
-                quality = playinfo['data']['quality'] or quality  # 0 indicates an error, fallback to best
+                current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
+                if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
+                    best_quality = playinfo['data']['accept_quality'][0]
             playinfos = []
             if playinfo is not None:
                 playinfos.append(playinfo)
             if playinfo_ is not None:
                 playinfos.append(playinfo_)
-            # get lower formats from API
+            # get alternative formats from API
             for qn in [80, 64, 32, 16]:
                 # automatic format for durl: qn=0
-                # For dash, qn does not matter
-                if qn < quality:
+                # for dash, qn does not matter
+                if current_quality is None or qn < current_quality:
                     api_url = self.bilibili_api(avid, cid, qn=qn)
                     api_content = get_content(api_url, headers=self.bilibili_headers())
                     api_playinfo = json.loads(api_content)
@@ -114,6 +125,12 @@ def prepare(self, **kwargs):
                         playinfos.append(api_playinfo)
                     else:
                         message = api_playinfo['data']['message']
+                elif best_quality is not None and current_quality < qn <= best_quality:
+                    api_url = self.bilibili_interface_api(cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo_data = json.loads(api_content)
+                    if api_playinfo_data.get('quality'):
+                        playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
             if not playinfos:
                 log.w(message)
                 # use bilibili error video instead

From 22b94849829c6f4f87e4c42f65767a3369b3c531 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 15:13:07 +0100
Subject: [PATCH 0784/1225] [bilibili] support /index_... page number

---
 src/you_get/extractors/bilibili.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 22a50e7b6f..2ba3fd0620 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -95,7 +95,8 @@ def prepare(self, **kwargs):
             # set video title
             self.title = initial_state['videoData']['title']
             # refine title for a specific part, if it is a multi-part video
-            p = int(match1(self.url, r'[\?&]p=(\d+)') or '1')  # use URL to decide p-number, not initial_state['p']
+            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
+                    '1')  # use URL to decide p-number, not initial_state['p']
             if pn > 1:
                 part = initial_state['videoData']['pages'][p - 1]['part']
                 self.title = '%s (P%s. %s)' % (self.title, p, part)

From fc94a5af1cb711ea64e62d29a1987bee972aaffc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 15:26:21 +0100
Subject: [PATCH 0785/1225] [bilibili] access interface more

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 2ba3fd0620..261cd37783 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -126,7 +126,7 @@ def prepare(self, **kwargs):
                         playinfos.append(api_playinfo)
                     else:
                         message = api_playinfo['data']['message']
-                elif best_quality is not None and current_quality < qn <= best_quality:
+                if best_quality is None or qn <= best_quality:
                     api_url = self.bilibili_interface_api(cid, qn=qn)
                     api_content = get_content(api_url, headers=self.bilibili_headers())
                     api_playinfo_data = json.loads(api_content)

From d675107e9796cb8114d4dcd5542068c06b9e7de3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 16:02:21 +0100
Subject: [PATCH 0786/1225] [bilibili] add stream type for quality: 48

---
 src/you_get/extractors/bilibili.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 261cd37783..3433d45ff5 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -19,10 +19,13 @@ class Bilibili(VideoExtractor):
          'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
         {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
+        {'id': 'hdmp4', 'quality': 48, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P (MP4)'},
         {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
         {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
          'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
+        # 'quality': 15?
     ]
 
     @staticmethod

From 51e31f0e53c8f479dd28e79dd79fbd7120991bbe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 16:26:30 +0100
Subject: [PATCH 0787/1225] [bilibili] support watchlater URLs

---
 src/you_get/extractors/bilibili.py | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 3433d45ff5..57fc4d5438 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -62,8 +62,15 @@ def prepare(self, **kwargs):
         #self.title = match1(html_content,
         #                    r'<h1 title="([^"]+)"')
 
+        # redirect: watchlater
+        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
+            avid = match1(self.url, r'/av(\d+)')
+            p = int(match1(self.url, r'/p(\d+)') or '1')
+            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
         # redirect: bangumi/play/ss -> bangumi/play/ep
-        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ss(\d+)', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url):
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             ep_id = initial_state['epList'][0]['id']
@@ -71,11 +78,11 @@ def prepare(self, **kwargs):
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
         # sort it out
-        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
-        elif re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
 
         # regular av video
@@ -264,11 +271,11 @@ def download_playlist_by_url(self, url, **kwargs):
         html_content = get_content(self.url, headers=self.bilibili_headers())
 
         # sort it out
-        if re.match(r'https?://(www)?\.bilibili\.com/bangumi/play/ep(\d+)', self.url):
+        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
-        elif re.match(r'https?://(www)?\.bilibili\.com/video/av(\d+)', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
 
         # regular av video

From 15bca0e50937af4fea14df2ee1f069a4dd033453 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 17:13:15 +0100
Subject: [PATCH 0788/1225] [bilibili] support vc videos

---
 src/you_get/extractors/bilibili.py | 51 ++++++++++++++++++++++++------
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 57fc4d5438..9cce1901dd 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -28,6 +28,17 @@ class Bilibili(VideoExtractor):
         # 'quality': 15?
     ]
 
+    @staticmethod
+    def height_to_quality(height):
+        if height <= 360:
+            return 16
+        elif height <= 480:
+            return 32
+        elif height <= 720:
+            return 64
+        else:
+            return 80
+
     @staticmethod
     def bilibili_headers(referer=None, cookie=None):
         # a reasonable UA
@@ -53,7 +64,11 @@ def bilibili_interface_api(cid, qn=0):
         appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
         params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
         chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
-        return 'http://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+
+    @staticmethod
+    def bilibili_vc_api(video_id):
+        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
@@ -82,6 +97,8 @@ def prepare(self, **kwargs):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
+        elif re.match(r'https?://vc\.?bilibili\.com/video/(\d+)', self.url):
+            sort = 'vc'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
 
@@ -211,15 +228,7 @@ def prepare(self, **kwargs):
 
             # DASH formats
             for video in data['result']['dash']['video']:
-                # convert height to quality code
-                if video['height'] == 360:
-                    quality = 16
-                elif video['height'] == 480:
-                    quality = 32
-                elif video['height'] == 720:
-                    quality = 64
-                elif video['height'] == 1080:
-                    quality = 80
+                quality = self.height_to_quality(video['height'])  # convert height to quality code
                 s = self.stream_qualities[quality]
                 format_id = 'dash-' + s['id']  # prefix
                 container = 'mp4'  # enforce MP4 container
@@ -239,6 +248,28 @@ def prepare(self, **kwargs):
                 self.dash_streams[format_id] = {'container': container, 'quality': desc,
                                                 'src': [[baseurl], [audio_baseurl]], 'size': size}
 
+        # vc video
+        elif sort == 'vc':
+            video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')
+            api_url = self.bilibili_vc_api(video_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_playinfo = json.loads(api_content)
+
+            # set video title
+            self.title = '%s (%s)' % (api_playinfo['data']['user']['name'], api_playinfo['data']['item']['id'])
+
+            height = api_playinfo['data']['item']['height']
+            quality = self.height_to_quality(height)  # convert height to quality code
+            s = self.stream_qualities[quality]
+            format_id = s['id']
+            container = 'mp4'  # enforce MP4 container
+            desc = s['desc']
+
+            playurl = api_playinfo['data']['item']['video_playurl']
+            size = int(api_playinfo['data']['item']['video_size'])
+
+            self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
+
 
         else:
             # NOT IMPLEMENTED

From 17d1597c415dbe1bfc22619afe8bbed83c624200 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 22:17:37 +0100
Subject: [PATCH 0789/1225] [bilibili] support durl in bangumi

---
 src/you_get/extractors/bilibili.py | 53 +++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 20 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9cce1901dd..642a3d7ed3 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -226,27 +226,40 @@ def prepare(self, **kwargs):
                 log.e(data['message'])
                 return
 
+            if 'durl' in data['result']:
+                quality = data['result']['quality']
+                format_id = self.stream_qualities[quality]['id']
+                container = self.stream_qualities[quality]['container'].lower()
+                desc = self.stream_qualities[quality]['desc']
+
+                src, size = [], 0
+                for durl in data['result']['durl']:
+                    src.append(durl['url'])
+                    size += durl['size']
+                self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
             # DASH formats
-            for video in data['result']['dash']['video']:
-                quality = self.height_to_quality(video['height'])  # convert height to quality code
-                s = self.stream_qualities[quality]
-                format_id = 'dash-' + s['id']  # prefix
-                container = 'mp4'  # enforce MP4 container
-                desc = s['desc']
-                audio_quality = s['audio_quality']
-                baseurl = video['baseUrl']
-                size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                # find matching audio track
-                audio_baseurl = data['result']['dash']['audio'][0]['baseUrl']
-                for audio in data['result']['dash']['audio']:
-                    if int(audio['id']) == audio_quality:
-                        audio_baseurl = audio['baseUrl']
-                        break
-                size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                'src': [[baseurl], [audio_baseurl]], 'size': size}
+            if 'dash' in data['result']:
+                for video in data['result']['dash']['video']:
+                    quality = self.height_to_quality(video['height'])  # convert height to quality code
+                    s = self.stream_qualities[quality]
+                    format_id = 'dash-' + s['id']  # prefix
+                    container = 'mp4'  # enforce MP4 container
+                    desc = s['desc']
+                    audio_quality = s['audio_quality']
+                    baseurl = video['baseUrl']
+                    size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    # find matching audio track
+                    audio_baseurl = data['result']['dash']['audio'][0]['baseUrl']
+                    for audio in data['result']['dash']['audio']:
+                        if int(audio['id']) == audio_quality:
+                            audio_baseurl = audio['baseUrl']
+                            break
+                    size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         # vc video
         elif sort == 'vc':

From cdb0d3d17097e1016b7868db1a8969891f7bb628 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 23:06:09 +0100
Subject: [PATCH 0790/1225] [bilibili] get alternative qualities for bangumi

---
 src/you_get/extractors/bilibili.py | 95 ++++++++++++++++++------------
 1 file changed, 56 insertions(+), 39 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 642a3d7ed3..00d095f5b1 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -208,58 +208,75 @@ def prepare(self, **kwargs):
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
 
-            # set video title
-            self.title = initial_state['h1Title']
-
             # warn if this bangumi has more than 1 video
             epn = len(initial_state['epList'])
             if epn > 1 and not kwargs.get('playlist'):
                 log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
 
+            # set video title
+            self.title = initial_state['h1Title']
+
+            # construct playinfos
             ep_id = initial_state['epInfo']['id']
             avid = initial_state['epInfo']['aid']
             cid = initial_state['epInfo']['cid']
+            playinfos = []
             api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
             api_content = get_content(api_url, headers=self.bilibili_headers())
-            data = json.loads(api_content)
-            if data['code'] < 0:  # error
-                log.e(data['message'])
+            api_playinfo = json.loads(api_content)
+            if api_playinfo['code'] == 0:  # success
+                playinfos.append(api_playinfo)
+            else:
+                log.e(api_playinfo['message'])
                 return
+            current_quality = api_playinfo['result']['quality']
+            # get alternative formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # for dash, qn does not matter
+                if qn != current_quality:
+                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
 
-            if 'durl' in data['result']:
-                quality = data['result']['quality']
-                format_id = self.stream_qualities[quality]['id']
-                container = self.stream_qualities[quality]['container'].lower()
-                desc = self.stream_qualities[quality]['desc']
+            for playinfo in playinfos:
+                if 'durl' in playinfo['result']:
+                    quality = playinfo['result']['quality']
+                    format_id = self.stream_qualities[quality]['id']
+                    container = self.stream_qualities[quality]['container'].lower()
+                    desc = self.stream_qualities[quality]['desc']
 
-                src, size = [], 0
-                for durl in data['result']['durl']:
-                    src.append(durl['url'])
-                    size += durl['size']
-                self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-            # DASH formats
-            if 'dash' in data['result']:
-                for video in data['result']['dash']['video']:
-                    quality = self.height_to_quality(video['height'])  # convert height to quality code
-                    s = self.stream_qualities[quality]
-                    format_id = 'dash-' + s['id']  # prefix
-                    container = 'mp4'  # enforce MP4 container
-                    desc = s['desc']
-                    audio_quality = s['audio_quality']
-                    baseurl = video['baseUrl']
-                    size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                    # find matching audio track
-                    audio_baseurl = data['result']['dash']['audio'][0]['baseUrl']
-                    for audio in data['result']['dash']['audio']:
-                        if int(audio['id']) == audio_quality:
-                            audio_baseurl = audio['baseUrl']
-                            break
-                    size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
+                    src, size = [], 0
+                    for durl in playinfo['result']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['result']:
+                    for video in playinfo['result']['dash']['video']:
+                        # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
+                        quality = self.height_to_quality(video['height'])  # convert height to quality code
+                        s = self.stream_qualities[quality]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['result']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['result']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
 
         # vc video
         elif sort == 'vc':

From 90d40dcd35ffb5e8bc7b46ee4d211c43c0b7e73c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Feb 2019 23:16:35 +0100
Subject: [PATCH 0791/1225] [bilibili] support old bangumi.bilibili.com/anime
 URLs

---
 src/you_get/extractors/bilibili.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 00d095f5b1..b01777713f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -85,7 +85,9 @@ def prepare(self, **kwargs):
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
         # redirect: bangumi/play/ss -> bangumi/play/ep
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url):
+        # redirect: bangumi.bilibili.com/anime -> bangumi/play/ep
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url) or \
+             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)/play', self.url):
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             ep_id = initial_state['epList'][0]['id']
@@ -101,6 +103,9 @@ def prepare(self, **kwargs):
             sort = 'vc'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
+        else:
+            log.e('[Error] Unsupported URL pattern.')
+            exit(1)
 
         # regular av video
         if sort == 'video':

From f1ab3f223df21cecb43765c5054637b58ba1f4d0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 00:00:21 +0100
Subject: [PATCH 0792/1225] [bilibili] support bangumi/media

---
 src/you_get/extractors/bilibili.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index b01777713f..21b55e772e 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -104,8 +104,8 @@ def prepare(self, **kwargs):
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
         else:
-            log.e('[Error] Unsupported URL pattern.')
-            exit(1)
+            self.download_playlist_by_url(self.url, **kwargs)
+            return
 
         # regular av video
         if sort == 'video':
@@ -341,8 +341,14 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
+            re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
+            sort = 'bangumi_md'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
+        else:
+            log.e('[Error] Unsupported URL pattern.')
+            exit(1)
 
         # regular av video
         if sort == 'video':
@@ -357,10 +363,24 @@ def download_playlist_by_url(self, url, **kwargs):
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['epList']), 0
             for ep in initial_state['epList']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 ep_id = ep['id']
                 epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
                 self.__class__().download_by_url(epurl, **kwargs)
+                sys.stdout.flush()
+
+        elif sort == 'bangumi_md':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['mediaInfo']['episodes']), 0
+            for ep in initial_state['mediaInfo']['episodes']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                ep_id = ep['ep_id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+                sys.stdout.flush()
 
 
 site = Bilibili()

From 2ed8aa2daed2ca0726eebcff6efeef49f90018df Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 00:54:40 +0100
Subject: [PATCH 0793/1225] [bilibili] support space channel

---
 src/you_get/extractors/bilibili.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 21b55e772e..2a06c5da29 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -70,6 +70,10 @@ def bilibili_interface_api(cid, qn=0):
     def bilibili_vc_api(video_id):
         return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
+    @staticmethod
+    def bilibili_space_channel_api(mid, cid, ps=128):
+        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
+
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
@@ -346,6 +350,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'bangumi_md'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
+            sort = 'space_channel'
         else:
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
@@ -382,6 +388,19 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_by_url(epurl, **kwargs)
                 sys.stdout.flush()
 
+        elif sort == 'space_channel':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
+            mid, cid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_channel_api(mid, cid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            channel_info = json.loads(api_content)
+            epn, i = len(channel_info['data']['list']['archives']), 0
+            for video in channel_info['data']['list']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+                sys.stdout.flush()
+
 
 site = Bilibili()
 download = site.download_by_url

From df74d4dd07ab7fbad3e3ca01592e26729ad53192 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 01:17:06 +0100
Subject: [PATCH 0794/1225] [bilibili] support space favlist

---
 src/you_get/extractors/bilibili.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 2a06c5da29..8c9cb44808 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -70,6 +70,10 @@ def bilibili_interface_api(cid, qn=0):
     def bilibili_vc_api(video_id):
         return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
+    @staticmethod
+    def bilibili_favlist_api(vmid, fid, ps=128):
+        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, ps)
+
     @staticmethod
     def bilibili_space_channel_api(mid, cid, ps=128):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
@@ -350,6 +354,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'bangumi_md'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
+            sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
             sort = 'space_channel'
         else:
@@ -388,6 +394,19 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_by_url(epurl, **kwargs)
                 sys.stdout.flush()
 
+        elif sort == 'space_favlist':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
+            vmid, fid = m.group(1), m.group(2)
+            api_url = self.bilibili_favlist_api(vmid, fid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            favlist_info = json.loads(api_content)
+            epn, i = len(favlist_info['data']['archives']), 0
+            for video in favlist_info['data']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+                sys.stdout.flush()
+
         elif sort == 'space_channel':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
             mid, cid = m.group(1), m.group(2)

From d9ed4f4a0ff8946a5707a905e107e81bcd28545a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 01:30:24 +0100
Subject: [PATCH 0795/1225] [bilibili] support space video

---
 src/you_get/extractors/bilibili.py | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 8c9cb44808..e1b2ad502a 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -71,11 +71,15 @@ def bilibili_vc_api(video_id):
         return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
     @staticmethod
-    def bilibili_favlist_api(vmid, fid, ps=128):
+    def bilibili_space_video_api(mid, ps=100):
+        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=1&pagesize=%s&order=0&jsonp=jsonp' % (mid, ps)
+
+    @staticmethod
+    def bilibili_space_favlist_api(vmid, fid, ps=100):
         return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, ps)
 
     @staticmethod
-    def bilibili_space_channel_api(mid, cid, ps=128):
+    def bilibili_space_channel_api(mid, cid, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
 
     def prepare(self, **kwargs):
@@ -354,6 +358,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'bangumi_md'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
+            sort = 'space_video'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
             sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
@@ -394,10 +400,23 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_by_url(epurl, **kwargs)
                 sys.stdout.flush()
 
+        elif sort == 'space_video':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
+            mid = m.group(1)
+            api_url = self.bilibili_space_video_api(mid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            videos_info = json.loads(api_content)
+            epn, i = len(videos_info['data']['vlist']), 0
+            for video in videos_info['data']['vlist']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+                sys.stdout.flush()
+
         elif sort == 'space_favlist':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
             vmid, fid = m.group(1), m.group(2)
-            api_url = self.bilibili_favlist_api(vmid, fid)
+            api_url = self.bilibili_space_favlist_api(vmid, fid)
             api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
             favlist_info = json.loads(api_content)
             epn, i = len(favlist_info['data']['archives']), 0

From 3a98e6a5cb93b718ee0e4bd2734759a364ed32e4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 02:09:56 +0100
Subject: [PATCH 0796/1225] [bilibili] adjust ordering

---
 src/you_get/extractors/bilibili.py | 56 +++++++++++++++---------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e1b2ad502a..41e256c236 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -67,20 +67,20 @@ def bilibili_interface_api(cid, qn=0):
         return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
 
     @staticmethod
-    def bilibili_vc_api(video_id):
-        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
-
-    @staticmethod
-    def bilibili_space_video_api(mid, ps=100):
-        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=1&pagesize=%s&order=0&jsonp=jsonp' % (mid, ps)
+    def bilibili_space_channel_api(mid, cid, ps=100):
+        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
 
     @staticmethod
     def bilibili_space_favlist_api(vmid, fid, ps=100):
         return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, ps)
 
     @staticmethod
-    def bilibili_space_channel_api(mid, cid, ps=100):
-        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
+    def bilibili_space_video_api(mid, ps=100):
+        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=1&pagesize=%s&order=0&jsonp=jsonp' % (mid, ps)
+
+    @staticmethod
+    def bilibili_vc_api(video_id):
+        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
@@ -358,12 +358,12 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'bangumi_md'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
-            sort = 'space_video'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
-            sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
             sort = 'space_channel'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
+            sort = 'space_favlist'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
+            sort = 'space_video'
         else:
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
@@ -400,14 +400,14 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_by_url(epurl, **kwargs)
                 sys.stdout.flush()
 
-        elif sort == 'space_video':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
-            mid = m.group(1)
-            api_url = self.bilibili_space_video_api(mid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            videos_info = json.loads(api_content)
-            epn, i = len(videos_info['data']['vlist']), 0
-            for video in videos_info['data']['vlist']:
+        elif sort == 'space_channel':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
+            mid, cid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_channel_api(mid, cid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            channel_info = json.loads(api_content)
+            epn, i = len(channel_info['data']['list']['archives']), 0
+            for video in channel_info['data']['list']['archives']:
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
@@ -426,14 +426,14 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_playlist_by_url(url, **kwargs)
                 sys.stdout.flush()
 
-        elif sort == 'space_channel':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
-            mid, cid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_channel_api(mid, cid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            channel_info = json.loads(api_content)
-            epn, i = len(channel_info['data']['list']['archives']), 0
-            for video in channel_info['data']['list']['archives']:
+        elif sort == 'space_video':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
+            mid = m.group(1)
+            api_url = self.bilibili_space_video_api(mid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            videos_info = json.loads(api_content)
+            epn, i = len(videos_info['data']['vlist']), 0
+            for video in videos_info['data']['vlist']:
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)

From 1c18a798902b9fca97bc688c2ffdc9a14f32474f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 03:14:42 +0100
Subject: [PATCH 0797/1225] [bilibili] support live

---
 src/you_get/extractors/bilibili.py | 47 ++++++++++++++++++++++++++++--
 1 file changed, 45 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 41e256c236..a5c96e54be 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -66,6 +66,18 @@ def bilibili_interface_api(cid, qn=0):
         chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
         return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
 
+    @staticmethod
+    def bilibili_live_api(cid):
+        return 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid=%s&quality=0&platform=web' % cid
+
+    @staticmethod
+    def bilibili_live_room_info_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id=%s' % room_id
+
+    @staticmethod
+    def bilibili_live_room_init_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
+
     @staticmethod
     def bilibili_space_channel_api(mid, cid, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
@@ -85,7 +97,10 @@ def bilibili_vc_api(video_id):
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
-        html_content = get_content(self.url, headers=self.bilibili_headers())
+        try:
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+        except:
+            html_content = ''  # live always returns 400 (why?)
         #self.title = match1(html_content,
         #                    r'<h1 title="([^"]+)"')
 
@@ -111,7 +126,9 @@ def prepare(self, **kwargs):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
-        elif re.match(r'https?://vc\.?bilibili\.com/video/(\d+)', self.url):
+        elif re.match(r'https?://live\.bilibili\.com/', self.url):
+            sort = 'live'
+        elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
             sort = 'vc'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
             sort = 'video'
@@ -317,6 +334,32 @@ def prepare(self, **kwargs):
 
             self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
 
+        # live
+        elif sort == 'live':
+            m = re.match(r'https?://live\.bilibili\.com/(\w+)', self.url)
+            short_id = m.group(1)
+            api_url = self.bilibili_live_room_init_api(short_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_init_info = json.loads(api_content)
+
+            room_id = room_init_info['data']['room_id']
+            api_url = self.bilibili_live_room_info_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_info = json.loads(api_content)
+
+            # set video title
+            self.title = room_info['data']['title'] + '.' + str(int(time.time()))
+
+            api_url = self.bilibili_live_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            video_info = json.loads(api_content)
+
+            durls = video_info['data']['durl']
+            playurl = durls[0]['url']
+            container = 'flv'  # enforce FLV container
+            self.streams['flv'] = {'container': container, 'quality': 'unknown',
+                                   'size': 0, 'src': [playurl]}
+
 
         else:
             # NOT IMPLEMENTED

From d13011c1255fd37bf83f2db36f90bcba4b25eaac Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 23:15:46 +0100
Subject: [PATCH 0798/1225] [bilibili] support audio

---
 src/you_get/extractors/bilibili.py | 35 +++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a5c96e54be..00c66e9f22 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -26,6 +26,7 @@ class Bilibili(VideoExtractor):
         {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
          'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
         # 'quality': 15?
+        {'id': 'mp4', 'quality': 0},
     ]
 
     @staticmethod
@@ -54,6 +55,14 @@ def bilibili_headers(referer=None, cookie=None):
     def bilibili_api(avid, cid, qn=0):
         return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
 
+    @staticmethod
+    def bilibili_audio_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/url?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
+
     @staticmethod
     def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
         return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
@@ -122,7 +131,9 @@ def prepare(self, **kwargs):
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
         # sort it out
-        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
+        if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
+            sort = 'audio'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
             sort = 'bangumi'
         elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
             sort = 'bangumi'
@@ -360,6 +371,28 @@ def prepare(self, **kwargs):
             self.streams['flv'] = {'container': container, 'quality': 'unknown',
                                    'size': 0, 'src': [playurl]}
 
+        # audio
+        elif sort == 'audio':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/au(\d+)', self.url)
+            sid = m.group(1)
+            api_url = self.bilibili_audio_info_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            song_info = json.loads(api_content)
+
+            # set audio title
+            self.title = song_info['data']['title']
+            self.lyric = song_info['data']['lyric']
+
+            api_url = self.bilibili_audio_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            audio_info = json.loads(api_content)
+
+            playurl = audio_info['data']['cdns'][0]
+            size = audio_info['data']['size']
+            container = 'mp4'  # enforce MP4 container
+            self.streams['mp4'] = {'container': container,
+                                   'size': size, 'src': [playurl]}
+
 
         else:
             # NOT IMPLEMENTED

From fb07bc56af7081c544b135fa1a50f0285f907be3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Feb 2019 23:43:05 +0100
Subject: [PATCH 0799/1225] [bilibili] support audio am playlists

---
 src/you_get/extractors/bilibili.py | 31 +++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 00c66e9f22..e8693606f6 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -63,6 +63,14 @@ def bilibili_audio_api(sid):
     def bilibili_audio_info_api(sid):
         return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
 
+    @staticmethod
+    def bilibili_audio_menu_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/menu/info?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_menu_song_api(sid, ps=100):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
+
     @staticmethod
     def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
         return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
@@ -393,11 +401,6 @@ def prepare(self, **kwargs):
             self.streams['mp4'] = {'container': container,
                                    'size': size, 'src': [playurl]}
 
-
-        else:
-            # NOT IMPLEMENTED
-            pass
-
     def extract(self, **kwargs):
         # set UA and referer for downloading
         headers = self.bilibili_headers(referer=self.url)
@@ -440,6 +443,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
             sort = 'space_video'
+        elif re.match(r'https?://(www\.)?bilibili\.com/audio/am(\d+)', self.url):
+            sort = 'audio_menu'
         else:
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
@@ -515,6 +520,22 @@ def download_playlist_by_url(self, url, **kwargs):
                 self.__class__().download_playlist_by_url(url, **kwargs)
                 sys.stdout.flush()
 
+        elif sort == 'audio_menu':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
+            sid = m.group(1)
+            #api_url = self.bilibili_audio_menu_info_api(sid)
+            #api_content = get_content(api_url, headers=self.bilibili_headers())
+            #menu_info = json.loads(api_content)
+            api_url = self.bilibili_audio_menu_song_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            menusong_info = json.loads(api_content)
+            epn, i = len(menusong_info['data']['data']), 0
+            for song in menusong_info['data']['data']:
+                i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
+                url = 'https://www.bilibili.com/audio/au%s' % song['id']
+                self.__class__().download_by_url(url, **kwargs)
+                sys.stdout.flush()
+
 
 site = Bilibili()
 download = site.download_by_url

From c0188253a0c141f139d015c857c6eb1544648279 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 00:35:52 +0100
Subject: [PATCH 0800/1225] [extractor] it's "danmaku"

---
 src/you_get/extractor.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 996d9a0633..bbe2af8d56 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -32,7 +32,7 @@ def __init__(self, *args):
         self.out = False
         self.ua = None
         self.referer = None
-        self.danmuku = None
+        self.danmaku = None
 
         if args:
             self.url = args[0]
@@ -231,7 +231,7 @@ def download(self, **kwargs):
                           merge=kwargs['merge'],
                           av=stream_id in self.dash_streams)
             if 'caption' not in kwargs or not kwargs['caption']:
-                print('Skipping captions or danmuku.')
+                print('Skipping captions or danmaku.')
                 return
             for lang in self.caption_tracks:
                 filename = '%s.%s.srt' % (get_filename(self.title), lang)
@@ -241,11 +241,11 @@ def download(self, **kwargs):
                           'w', encoding='utf-8') as x:
                     x.write(srt)
                 print('Done.')
-            if self.danmuku is not None and not dry_run:
+            if self.danmaku is not None and not dry_run:
                 filename = '{}.cmt.xml'.format(get_filename(self.title))
                 print('Downloading {} ...\n'.format(filename))
                 with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
-                    fp.write(self.danmuku)
+                    fp.write(self.danmaku)
 
             # For main_dev()
             #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])

From 23d89eb6e25c0dbd393592dd41ff962864c14ae4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 00:44:15 +0100
Subject: [PATCH 0801/1225] [bilibili] get danmaku

---
 src/you_get/extractors/bilibili.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e8693606f6..9941367b4c 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -256,6 +256,9 @@ def prepare(self, **kwargs):
                         self.dash_streams[format_id] = {'container': container, 'quality': desc,
                                                         'src': [[baseurl], [audio_baseurl]], 'size': size}
 
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
         # bangumi
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
@@ -389,7 +392,9 @@ def prepare(self, **kwargs):
 
             # set audio title
             self.title = song_info['data']['title']
-            self.lyric = song_info['data']['lyric']
+
+            lyric = song_info['data']['lyric']
+            # TODO: download lyrics
 
             api_url = self.bilibili_audio_api(sid)
             api_content = get_content(api_url, headers=self.bilibili_headers())

From 192d54ad6a6f291884c0782bc7975553db9972b8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 01:11:13 +0100
Subject: [PATCH 0802/1225] [extractor] download best-quality DASH stream if
 FFmpeg is installed and not using a player (revive 12de1b4)

---
 src/you_get/extractor.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index bbe2af8d56..e5efaf6d64 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 
-from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy, get_content, dry_run
+from .common import match1, maybe_print, download_urls, get_filename, parse_host, set_proxy, unset_proxy, get_content, dry_run, player
 from .common import print_more_compatible as print
 from .util import log
 from . import json_output
@@ -196,10 +196,13 @@ def download(self, **kwargs):
             else:
                 # Download stream with the best quality
                 from .processor.ffmpeg import has_ffmpeg_installed
-                if self.streams_sorted:
-                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                if has_ffmpeg_installed() and player is None and self.dash_streams or not self.streams_sorted:
+                    #stream_id = list(self.dash_streams)[-1]
+                    itags = sorted(self.dash_streams,
+                                   key=lambda i: -self.dash_streams[i]['size'])
+                    stream_id = itags[0]
                 else:
-                    stream_id = list(self.dash_streams)[-1]
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
 
             if 'index' not in kwargs:
                 self.p(stream_id)

From 1f271aeec9b9bbfb8d087d5b277d99bc7a5ea05f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 01:17:21 +0100
Subject: [PATCH 0803/1225] [extractor] flush output after printing each video
 info

---
 src/you_get/extractor.py           | 5 +++++
 src/you_get/extractors/bilibili.py | 6 ------
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index e5efaf6d64..13026bcded 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -5,6 +5,7 @@
 from .util import log
 from . import json_output
 import os
+import sys
 
 class Extractor():
     def __init__(self, *args):
@@ -130,6 +131,8 @@ def p_i(self, stream_id):
         print("        url:         %s" % self.url)
         print()
 
+        sys.stdout.flush()
+
     def p(self, stream_id=None):
         maybe_print("site:                %s" % self.__class__.name)
         maybe_print("title:               %s" % self.title)
@@ -165,6 +168,8 @@ def p(self, stream_id=None):
                 print("    - lang:          {}".format(i['lang']))
                 print("      download-url:  {}\n".format(i['url']))
 
+        sys.stdout.flush()
+
     def p_playlist(self, stream_id=None):
         maybe_print("site:                %s" % self.__class__.name)
         print("playlist:            %s" % self.title)
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9941367b4c..e926a07f4d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -473,7 +473,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 ep_id = ep['id']
                 epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
                 self.__class__().download_by_url(epurl, **kwargs)
-                sys.stdout.flush()
 
         elif sort == 'bangumi_md':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
@@ -484,7 +483,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 ep_id = ep['ep_id']
                 epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
                 self.__class__().download_by_url(epurl, **kwargs)
-                sys.stdout.flush()
 
         elif sort == 'space_channel':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
@@ -497,7 +495,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
-                sys.stdout.flush()
 
         elif sort == 'space_favlist':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
@@ -510,7 +507,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
-                sys.stdout.flush()
 
         elif sort == 'space_video':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
@@ -523,7 +519,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
-                sys.stdout.flush()
 
         elif sort == 'audio_menu':
             m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
@@ -539,7 +534,6 @@ def download_playlist_by_url(self, url, **kwargs):
                 i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
                 url = 'https://www.bilibili.com/audio/au%s' % song['id']
                 self.__class__().download_by_url(url, **kwargs)
-                sys.stdout.flush()
 
 
 site = Bilibili()

From 3acbec98cedcfecd6631f50f0f6f6fdffd265f0c Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Wed, 20 Feb 2019 13:45:29 +0100
Subject: [PATCH 0804/1225] [bilibili] cache DASH audio size

---
 src/you_get/extractors/bilibili.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e926a07f4d..a256421f36 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -235,6 +235,7 @@ def prepare(self, **kwargs):
 
                 # DASH formats
                 if 'dash' in playinfo['data']:
+                    audio_size_cache = {}
                     for video in playinfo['data']['dash']['video']:
                         # prefer the latter codecs!
                         s = self.stream_qualities[video['id']]
@@ -251,7 +252,9 @@ def prepare(self, **kwargs):
                             if int(audio['id']) == audio_quality:
                                 audio_baseurl = audio['baseUrl']
                                 break
-                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                        if not audio_size_cache.get(audio_quality, False):
+                            audio_size_cache[audio_quality] = url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                        size += audio_size_cache[audio_quality]
 
                         self.dash_streams[format_id] = {'container': container, 'quality': desc,
                                                         'src': [[baseurl], [audio_baseurl]], 'size': size}

From 931101c427e8b4ac88de2ef7f470aacf942a0fb1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 13:49:35 +0100
Subject: [PATCH 0805/1225] [bilibili] get danmaku for bangumi

---
 src/you_get/extractors/bilibili.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a256421f36..a601aad676 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -337,6 +337,9 @@ def prepare(self, **kwargs):
                         self.dash_streams[format_id] = {'container': container, 'quality': desc,
                                                         'src': [[baseurl], [audio_baseurl]], 'size': size}
 
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
         # vc video
         elif sort == 'vc':
             video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')

From d3ad3d96bee76bdb2961a2958791363083da7a67 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 14:18:31 +0100
Subject: [PATCH 0806/1225] Revert "[bilibili] assume container to be mp4
 (#2681)"

This reverts commit 6e89b8a1e194f3e96451411c50d4d950b5a88814.
---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index dbab756341..4685311813 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -229,8 +229,8 @@ def entry(self, **kwargs):
             playinfo_text = match1(self.page, r'__playinfo__=(.*?)<')
             playinfo = json.loads(playinfo_text)
             url0 = playinfo['data']['durl'][0]['url']
-            _, _, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
-            self.streams['flv'] = {'url': url0, 'container': 'mp4', 'size': size, 'src': [url0]}
+            _, ext, size = url_info(url0, headers={'referer': self.url, 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'})
+            self.streams['flv'] = {'url': url0, 'container': ext, 'size': size, 'src': [url0]}
         else:
             # flashvars?
             flashvars = re.search(r'flashvars="([^"]+)"', self.page).group(1)

From 11184c2552089ce6b48552800e0e99ac1c6b8973 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 14:58:17 +0100
Subject: [PATCH 0807/1225] [common] get_content(): ignore decoding errors

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9735a30f6a..137c49331e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -442,7 +442,7 @@ def get_content(url, headers={}, decoded=True):
             response.getheader('Content-Type', ''), r'charset=([\w-]+)'
         )
         if charset is not None:
-            data = data.decode(charset)
+            data = data.decode(charset, 'ignore')
         else:
             data = data.decode('utf-8', 'ignore')
 

From 96a25273a1c7526104718c256857e2387fc62d06 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 15:07:53 +0100
Subject: [PATCH 0808/1225] [bilibili] support audio lyrics

---
 src/you_get/extractor.py           | 10 ++++++++++
 src/you_get/extractors/bilibili.py |  4 ++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index 13026bcded..c4315935e7 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -34,6 +34,7 @@ def __init__(self, *args):
         self.ua = None
         self.referer = None
         self.danmaku = None
+        self.lyrics = None
 
         if args:
             self.url = args[0]
@@ -238,9 +239,11 @@ def download(self, **kwargs):
                           output_dir=kwargs['output_dir'],
                           merge=kwargs['merge'],
                           av=stream_id in self.dash_streams)
+
             if 'caption' not in kwargs or not kwargs['caption']:
                 print('Skipping captions or danmaku.')
                 return
+
             for lang in self.caption_tracks:
                 filename = '%s.%s.srt' % (get_filename(self.title), lang)
                 print('Saving %s ... ' % filename, end="", flush=True)
@@ -249,12 +252,19 @@ def download(self, **kwargs):
                           'w', encoding='utf-8') as x:
                     x.write(srt)
                 print('Done.')
+
             if self.danmaku is not None and not dry_run:
                 filename = '{}.cmt.xml'.format(get_filename(self.title))
                 print('Downloading {} ...\n'.format(filename))
                 with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
                     fp.write(self.danmaku)
 
+            if self.lyrics is not None and not dry_run:
+                filename = '{}.lrc'.format(get_filename(self.title))
+                print('Downloading {} ...\n'.format(filename))
+                with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
+                    fp.write(self.lyrics)
+
             # For main_dev()
             #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
         keep_obj = kwargs.get('keep_obj', False)
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a601aad676..990aa4f365 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -399,8 +399,8 @@ def prepare(self, **kwargs):
             # set audio title
             self.title = song_info['data']['title']
 
-            lyric = song_info['data']['lyric']
-            # TODO: download lyrics
+            # get lyrics
+            self.lyrics = get_content(song_info['data']['lyric'])
 
             api_url = self.bilibili_audio_api(sid)
             api_content = get_content(api_url, headers=self.bilibili_headers())

From c55a1ea4893bf198bcb521c9ba73ea4969869b81 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 16:14:16 +0100
Subject: [PATCH 0809/1225] purge dead sites

---
 src/you_get/common.py              |  4 --
 src/you_get/extractors/__init__.py |  3 -
 src/you_get/extractors/dilidili.py | 89 ------------------------------
 src/you_get/extractors/fantasy.py  | 54 ------------------
 src/you_get/extractors/huaban.py   | 85 ----------------------------
 src/you_get/extractors/quanmin.py  | 28 ----------
 6 files changed, 263 deletions(-)
 delete mode 100644 src/you_get/extractors/dilidili.py
 delete mode 100644 src/you_get/extractors/fantasy.py
 delete mode 100644 src/you_get/extractors/huaban.py
 delete mode 100644 src/you_get/extractors/quanmin.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 137c49331e..1d1088daa6 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -37,13 +37,11 @@
     'cbs'              : 'cbs',
     'coub'             : 'coub',
     'dailymotion'      : 'dailymotion',
-    'dilidili'         : 'dilidili',
     'douban'           : 'douban',
     'douyin'           : 'douyin',
     'douyu'            : 'douyutv',
     'ehow'             : 'ehow',
     'facebook'         : 'facebook',
-    'fantasy'          : 'fantasy',
     'fc2'              : 'fc2video',
     'flickr'           : 'flickr',
     'freesound'        : 'freesound',
@@ -51,7 +49,6 @@
     'google'           : 'google',
     'giphy'            : 'giphy',
     'heavy-music'      : 'heavymusic',
-    'huaban'           : 'huaban',
     'huomao'           : 'huomaotv',
     'iask'             : 'sina',
     'icourses'         : 'icourses',
@@ -94,7 +91,6 @@
     'pptv'             : 'pptv',
     'qingting'         : 'qingting',
     'qq'               : 'qq',
-    'quanmin'          : 'quanmin',
     'showroom-live'    : 'showroom',
     'sina'             : 'sina',
     'smgbb'            : 'bilibili',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index d2c4c7b718..0c4cccc72d 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -13,20 +13,17 @@
 from .cntv import *
 from .coub import *
 from .dailymotion import *
-from .dilidili import *
 from .douban import *
 from .douyin import *
 from .douyutv import *
 from .ehow import *
 from .facebook import *
-from .fantasy import *
 from .fc2video import *
 from .flickr import *
 from .freesound import *
 from .funshion import *
 from .google import *
 from .heavymusic import *
-from .huaban import *
 from .icourses import *
 from .ifeng import *
 from .imgur import *
diff --git a/src/you_get/extractors/dilidili.py b/src/you_get/extractors/dilidili.py
deleted file mode 100644
index f7b5922def..0000000000
--- a/src/you_get/extractors/dilidili.py
+++ /dev/null
@@ -1,89 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['dilidili_download']
-
-from ..common import *
-from .ckplayer import ckplayer_download
-
-headers = {
-    'DNT': '1',
-    'Accept-Encoding': 'gzip, deflate, sdch, br',
-    'Accept-Language': 'en-CA,en;q=0.8,en-US;q=0.6,zh-CN;q=0.4,zh;q=0.2',
-    'Upgrade-Insecure-Requests': '1',
-    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-    'Cache-Control': 'max-age=0',
-    'Referer': 'http://www.dilidili.com/',
-    'Connection': 'keep-alive',
-    'Save-Data': 'on',
-}
-
-#----------------------------------------------------------------------
-def dilidili_parser_data_to_stream_types(typ ,vid ,hd2 ,sign, tmsign, ulk):
-    """->list"""
-    another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
-    parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
-    html = get_content(another_url, headers=headers)
-    
-    info = re.search(r'(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})(\{[^{]+\})', html).groups()
-    info = [i.strip('{}').split('->') for i in info]
-    info = {i[0]: i [1] for i in info}
-    
-    stream_types = []
-    for i in zip(info['deft'].split('|'), info['defa'].split('|')):
-        stream_types.append({'id': str(i[1][-1]), 'container': 'mp4', 'video_profile': i[0]})
-    return stream_types
-
-#----------------------------------------------------------------------
-def dilidili_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    global headers
-    re_str = r'http://www.dilidili.com/watch\S+'
-    if re.match(r'http://www.dilidili.wang', url):
-        re_str = r'http://www.dilidili.wang/watch\S+'
-        headers['Referer'] = 'http://www.dilidili.wang/'
-    elif re.match(r'http://www.dilidili.mobi', url):
-        re_str = r'http://www.dilidili.mobi/watch\S+'
-        headers['Referer'] = 'http://www.dilidili.mobi/'
-
-    if re.match(re_str, url):
-        html = get_content(url)
-        title = match1(html, r'<title>(.+)丨(.+)</title>')  #title
-        
-        # player loaded via internal iframe
-        frame_url = re.search(r'<iframe src=\"(.+?)\"', html).group(1)
-        logging.debug('dilidili_download: %s' % frame_url)
-        
-        #https://player.005.tv:60000/?vid=a8760f03fd:a04808d307&v=yun&sign=a68f8110cacd892bc5b094c8e5348432
-        html = get_content(frame_url, headers=headers, decoded=False).decode('utf-8')
-        
-        match = re.search(r'(.+?)var video =(.+?);', html)
-        vid = match1(html, r'var vid="(.+)"')
-        hd2 = match1(html, r'var hd2="(.+)"')
-        typ = match1(html, r'var typ="(.+)"')
-        sign = match1(html, r'var sign="(.+)"')
-        tmsign = match1(html, r'tmsign=([A-Za-z0-9]+)')
-        ulk =  match1(html, r'var ulk="(.+)"')
-
-        # here s the parser...
-        stream_types = dilidili_parser_data_to_stream_types(typ, vid, hd2, sign, tmsign, ulk)
-        
-        #get best
-        best_id = max([i['id'] for i in stream_types])
-        
-        parse_url = 'http://player.005.tv/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = best_id, sign = sign, tmsign = tmsign, ulk = ulk)
-        
-        another_url = 'https://newplayer.jfrft.com/parse.php?xmlurl=null&type={typ}&vid={vid}&hd={hd2}&sign={sign}&tmsign={tmsign}&userlink={ulk}'.format(typ = typ, vid = vid, hd2 = hd2, sign = sign, tmsign = tmsign, ulk = ulk)
-
-        ckplayer_download(another_url, output_dir, merge, info_only, is_xml = True, title = title, headers = headers)
-
-        #type_ = ''
-        #size = 0
-
-        #type_, ext, size = url_info(url)
-        #print_info(site_info, title, type_, size)
-        #if not info_only:
-            #download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
-
-site_info = "dilidili"
-download = dilidili_download
-download_playlist = playlist_not_supported('dilidili')
diff --git a/src/you_get/extractors/fantasy.py b/src/you_get/extractors/fantasy.py
deleted file mode 100644
index 3c7bee255c..0000000000
--- a/src/you_get/extractors/fantasy.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['fantasy_download']
-
-from ..common import *
-import json
-import random
-from urllib.parse import urlparse, parse_qs
-
-
-def fantasy_download_by_id_channelId(id = 0, channelId = 0, output_dir = '.', merge = True, info_only = False,
-                                     **kwargs):
-    api_url = 'http://www.fantasy.tv/tv/playDetails.action?' \
-              'myChannelId=1&id={id}&channelId={channelId}&t={t}'.format(id = id,
-                                                                         channelId = channelId,
-                                                                         t = str(random.random())
-                                                                         )
-    html = get_content(api_url)
-    html = json.loads(html)
-
-    if int(html['status']) != 100000:
-        raise Exception('API error!')
-
-    title = html['data']['tv']['title']
-
-    video_url = html['data']['tv']['videoPath']
-    headers = fake_headers.copy()
-    headers['Referer'] = api_url
-    type, ext, size = url_info(video_url, headers=headers)
-
-    print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([video_url], title, ext, size, output_dir, merge = merge, headers = headers)
-
-
-def fantasy_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    if 'fantasy.tv' not in url:
-        raise Exception('Wrong place!')
-
-    q = parse_qs(urlparse(url).query)
-
-    if 'tvId' not in q or 'channelId' not in q:
-        raise Exception('No enough arguments!')
-
-    tvId = q['tvId'][0]
-    channelId = q['channelId'][0]
-
-    fantasy_download_by_id_channelId(id = tvId, channelId = channelId, output_dir = output_dir, merge = merge,
-                                     info_only = info_only, **kwargs)
-
-
-site_info = "fantasy.tv"
-download = fantasy_download
-download_playlist = playlist_not_supported('fantasy.tv')
diff --git a/src/you_get/extractors/huaban.py b/src/you_get/extractors/huaban.py
deleted file mode 100644
index 8acf938bfa..0000000000
--- a/src/you_get/extractors/huaban.py
+++ /dev/null
@@ -1,85 +0,0 @@
-#!/usr/bin/env python
-
-import json
-import os
-import re
-import math
-import traceback
-import urllib.parse as urlparse
-
-from ..common import *
-
-__all__ = ['huaban_download']
-
-site_info = '花瓣 (Huaban)'
-
-LIMIT = 100
-
-
-class Board:
-    def __init__(self, title, pins):
-        self.title = title
-        self.pins = pins
-        self.pin_count = len(pins)
-
-
-class Pin:
-    host = 'http://img.hb.aicdn.com/'
-
-    def __init__(self, pin_json):
-        img_file = pin_json['file']
-        self.id = str(pin_json['pin_id'])
-        self.url = urlparse.urljoin(self.host, img_file['key'])
-        self.ext = img_file['type'].split('/')[-1]
-
-
-def construct_url(url, **params):
-    param_str = urlparse.urlencode(params)
-    return url + '?' + param_str
-
-
-def extract_json_data(url, **params):
-    url = construct_url(url, **params)
-    html = get_content(url, headers=fake_headers)
-    json_string = match1(html, r'app.page\["board"\] = (.*?});')
-    json_data = json.loads(json_string)
-    return json_data
-
-
-def extract_board_data(url):
-    json_data = extract_json_data(url, limit=LIMIT)
-    pin_list = json_data['pins']
-    title = json_data['title']
-    pin_count = json_data['pin_count']
-    pin_count -= len(pin_list)
-
-    while pin_count > 0:
-        json_data = extract_json_data(url, max=pin_list[-1]['pin_id'],
-                                      limit=LIMIT)
-        pins = json_data['pins']
-        pin_list += pins
-        pin_count -= len(pins)
-
-    return Board(title, list(map(Pin, pin_list)))
-
-
-def huaban_download_board(url, output_dir, **kwargs):
-    kwargs['merge'] = False
-    board = extract_board_data(url)
-    output_dir = os.path.join(output_dir, board.title)
-    print_info(site_info, board.title, 'jpg', float('Inf'))
-    for pin in board.pins:
-        download_urls([pin.url], pin.id, pin.ext, float('Inf'),
-                      output_dir=output_dir, faker=True, **kwargs)
-
-
-def huaban_download(url, output_dir='.', **kwargs):
-    if re.match(r'http://huaban\.com/boards/\d+/', url):
-        huaban_download_board(url, output_dir, **kwargs)
-    else:
-        print('Only board (画板) pages are supported currently')
-        print('ex: http://huaban.com/boards/12345678/')
-
-
-download = huaban_download
-download_playlist = playlist_not_supported("huaban")
diff --git a/src/you_get/extractors/quanmin.py b/src/you_get/extractors/quanmin.py
deleted file mode 100644
index 7428d1cbf0..0000000000
--- a/src/you_get/extractors/quanmin.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['quanmin_download']
-
-from ..common import *
-import json
-
-def quanmin_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    roomid = url.split('/')[3].split('?')[0]
-
-    json_request_url = 'http://m.quanmin.tv/json/rooms/{}/noinfo6.json'.format(roomid)
-    content = get_html(json_request_url)
-    data = json.loads(content)
-
-    title = data["title"]
-
-    if not data["play_status"]:
-        raise ValueError("The live stream is not online!")
-        
-    real_url = data["live"]["ws"]["flv"]["5"]["src"]
-
-    print_info(site_info, title, 'flv', float('inf'))
-    if not info_only:
-        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
-
-site_info = "quanmin.tv"
-download = quanmin_download
-download_playlist = playlist_not_supported('quanmin')

From e5fc0f2780a4d5b08a6f34662420cd0337de4804 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 16:42:24 +0100
Subject: [PATCH 0810/1225] update README.md

---
 README.md | 100 +++++++++++++++++++++++++++++++-----------------------
 1 file changed, 57 insertions(+), 43 deletions(-)

diff --git a/README.md b/README.md
index f524c60d92..6a9746aeee 100644
--- a/README.md
+++ b/README.md
@@ -49,10 +49,10 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 ### Prerequisites
 
-The following dependencies are required and must be installed separately, unless you are using a pre-built package or chocolatey on Windows:
+The following dependencies are necessary:
 
-* **[Python 3](https://www.python.org/downloads/)**
-* **[FFmpeg](https://www.ffmpeg.org/)** (strongly recommended) or [Libav](https://libav.org/)
+* **[Python](https://www.python.org/downloads/)**  3.2 or above
+* **[FFmpeg](https://www.ffmpeg.org/)** 1.0 or above
 * (Optional) [RTMPDump](https://rtmpdump.mplayerhq.hu/)
 
 ### Option 1: Install via pip
@@ -61,17 +61,13 @@ The official release of `you-get` is distributed on [PyPI](https://pypi.python.o
 
     $ pip3 install you-get
 
-### Option 2: Install via [Antigen](https://github.com/zsh-users/antigen)
+### Option 2: Install via [Antigen](https://github.com/zsh-users/antigen) (for Zsh users)
 
 Add the following line to your `.zshrc`:
 
     antigen bundle soimort/you-get
 
-### Option 3: Use a pre-built package (Windows only)
-
-Download the `exe` (standalone) or `7z` (all dependencies included) from: <https://github.com/soimort/you-get/releases/latest>.
-
-### Option 4: Download from GitHub
+### Option 3: Download from GitHub
 
 You may either download the [stable](https://github.com/soimort/you-get/archive/master.zip) (identical with the latest release on PyPI) or the [develop](https://github.com/soimort/you-get/archive/develop.zip) (more hotfixes, unstable features) branch of `you-get`. Unzip it, and put the directory containing the `you-get` script into your `PATH`.
 
@@ -89,7 +85,7 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
-### Option 5: Git clone
+### Option 4: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.
 
@@ -99,13 +95,7 @@ $ git clone git://github.com/soimort/you-get.git
 
 Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
 
-### Option 6: Using [Chocolatey](https://chocolatey.org/) (Windows only)
-
-```
-> choco install you-get
-```
-
-### Option 7: Homebrew (Mac only)
+### Option 5: Homebrew (Mac only)
 
 You can install `you-get` easily via:
 
@@ -113,7 +103,7 @@ You can install `you-get` easily via:
 $ brew install you-get
 ```
 
-### Option 8: pkg (FreeBSD only)
+### Option 6: pkg (FreeBSD only)
 
 You can install `you-get` easily via:
 
@@ -139,12 +129,6 @@ or download the latest release via:
 $ you-get https://github.com/soimort/you-get/archive/master.zip
 ```
 
-or use [chocolatey package manager](https://chocolatey.org):
-
-```
-> choco upgrade you-get
-```
-
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
 
 ```
@@ -162,22 +146,54 @@ $ you-get -i 'https://www.youtube.com/watch?v=jNQXAC9IVRw'
 site:                YouTube
 title:               Me at the zoo
 streams:             # Available quality and codecs
+    [ DASH ] ____________________________________
+    - itag:          242
+      container:     webm
+      quality:       320x240
+      size:          0.6 MiB (618358 bytes)
+    # download-with: you-get --itag=242 [URL]
+
+    - itag:          395
+      container:     mp4
+      quality:       320x240
+      size:          0.5 MiB (550743 bytes)
+    # download-with: you-get --itag=395 [URL]
+
+    - itag:          133
+      container:     mp4
+      quality:       320x240
+      size:          0.5 MiB (498558 bytes)
+    # download-with: you-get --itag=133 [URL]
+
+    - itag:          278
+      container:     webm
+      quality:       192x144
+      size:          0.4 MiB (392857 bytes)
+    # download-with: you-get --itag=278 [URL]
+
+    - itag:          160
+      container:     mp4
+      quality:       192x144
+      size:          0.4 MiB (370882 bytes)
+    # download-with: you-get --itag=160 [URL]
+
+    - itag:          394
+      container:     mp4
+      quality:       192x144
+      size:          0.4 MiB (367261 bytes)
+    # download-with: you-get --itag=394 [URL]
+
     [ DEFAULT ] _________________________________
     - itag:          43
       container:     webm
       quality:       medium
-      size:          0.5 MiB (564215 bytes)
+      size:          0.5 MiB (568748 bytes)
     # download-with: you-get --itag=43 [URL]
 
     - itag:          18
       container:     mp4
-      quality:       medium
-    # download-with: you-get --itag=18 [URL]
-
-    - itag:          5
-      container:     flv
       quality:       small
-    # download-with: you-get --itag=5 [URL]
+    # download-with: you-get --itag=18 [URL]
 
     - itag:          36
       container:     3gp
@@ -190,23 +206,24 @@ streams:             # Available quality and codecs
     # download-with: you-get --itag=17 [URL]
 ```
 
-The format marked with `DEFAULT` is the one you will get by default. If that looks cool to you, download it:
+By default, the one on the top is the one you will get. If that looks cool to you, download it:
 
 ```
 $ you-get 'https://www.youtube.com/watch?v=jNQXAC9IVRw'
 site:                YouTube
 title:               Me at the zoo
 stream:
-    - itag:          43
+    - itag:          242
       container:     webm
-      quality:       medium
-      size:          0.5 MiB (564215 bytes)
-    # download-with: you-get --itag=43 [URL]
+      quality:       320x240
+      size:          0.6 MiB (618358 bytes)
+    # download-with: you-get --itag=242 [URL]
 
-Downloading zoo.webm ...
-100.0% (  0.5/0.5  MB) ├████████████████████████████████████████┤[1/1]    7 MB/s
+Downloading Me at the zoo.webm ...
+ 100% (  0.6/  0.6MB) ├██████████████████████████████████████████████████████████████████████████████┤[2/2]    2 MB/s
+Merging video parts... Merged into Me at the zoo.webm
 
-Saving Me at the zoo.en.srt ...Done.
+Saving Me at the zoo.en.srt ... Done.
 ```
 
 (If a YouTube video has any closed captions, they will be downloaded together with the video file, in SubRip subtitle format.)
@@ -306,7 +323,7 @@ However, the system proxy setting (i.e. the environment variable `http_proxy`) i
 
 ### Watch a video
 
-Use the `--player`/`-p` option to feed the video into your media player of choice, e.g. `mplayer` or `vlc`, instead of downloading it:
+Use the `--player`/`-p` option to feed the video into your media player of choice, e.g. `mpv` or `vlc`, instead of downloading it:
 
 ```
 $ you-get -p vlc 'https://www.youtube.com/watch?v=jNQXAC9IVRw'
@@ -386,7 +403,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Baidu<br/>百度贴吧** | <http://tieba.baidu.com/> |✓|✓| |
 | 爆米花网 | <http://www.baomihua.com/>     |✓| | |
 | **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
-| Dilidili | <http://www.dilidili.com/>     |✓| | |
 | 豆瓣     | <http://www.douban.com/>       |✓| |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
 | Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
@@ -415,11 +431,9 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
 | 央视网   | <http://www.cntv.cn/>          |✓| | |
-| 花瓣     | <http://huaban.com/>           | |✓| |
 | Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
 | 芒果TV   | <http://www.mgtv.com/>         |✓| | |
 | 火猫TV   | <http://www.huomao.com/>       |✓| | |
-| 全民直播 | <http://www.quanmin.tv/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |

From f3c33870f59471b3c3f950175c5366fd42c5f89f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Feb 2019 16:43:07 +0100
Subject: [PATCH 0811/1225] version 0.4.1256

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index f2b279b1f3..663937491d 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1210'
+__version__ = '0.4.1256'

From e1ca6b0ec21e4474092e4481be41e7ee41991b1f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 26 Feb 2019 15:24:53 +0100
Subject: [PATCH 0812/1225] [bilibili] handle paging in space (close #2685)

---
 src/you_get/extractors/bilibili.py | 48 ++++++++++++++++++++----------
 1 file changed, 32 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 990aa4f365..cddd9d6f18 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -96,16 +96,16 @@ def bilibili_live_room_init_api(room_id):
         return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
 
     @staticmethod
-    def bilibili_space_channel_api(mid, cid, ps=100):
-        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (mid, cid, ps)
+    def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
 
     @staticmethod
-    def bilibili_space_favlist_api(vmid, fid, ps=100):
-        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=1&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, ps)
+    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
 
     @staticmethod
-    def bilibili_space_video_api(mid, ps=100):
-        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=1&pagesize=%s&order=0&jsonp=jsonp' % (mid, ps)
+    def bilibili_space_video_api(mid, pn=1, ps=100):
+        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
 
     @staticmethod
     def bilibili_vc_api(video_id):
@@ -496,6 +496,8 @@ def download_playlist_by_url(self, url, **kwargs):
             api_url = self.bilibili_space_channel_api(mid, cid)
             api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
             channel_info = json.loads(api_content)
+            # TBD: channel of more than 100 videos
+
             epn, i = len(channel_info['data']['list']['archives']), 0
             for video in channel_info['data']['list']['archives']:
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
@@ -508,11 +510,18 @@ def download_playlist_by_url(self, url, **kwargs):
             api_url = self.bilibili_space_favlist_api(vmid, fid)
             api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
             favlist_info = json.loads(api_content)
-            epn, i = len(favlist_info['data']['archives']), 0
-            for video in favlist_info['data']['archives']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                self.__class__().download_playlist_by_url(url, **kwargs)
+            pc = favlist_info['data']['pagecount']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                favlist_info = json.loads(api_content)
+
+                epn, i = len(favlist_info['data']['archives']), 0
+                for video in favlist_info['data']['archives']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
 
         elif sort == 'space_video':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
@@ -520,11 +529,18 @@ def download_playlist_by_url(self, url, **kwargs):
             api_url = self.bilibili_space_video_api(mid)
             api_content = get_content(api_url, headers=self.bilibili_headers())
             videos_info = json.loads(api_content)
-            epn, i = len(videos_info['data']['vlist']), 0
-            for video in videos_info['data']['vlist']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                self.__class__().download_playlist_by_url(url, **kwargs)
+            pc = videos_info['data']['pages']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_video_api(mid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                videos_info = json.loads(api_content)
+
+                epn, i = len(videos_info['data']['vlist']), 0
+                for video in videos_info['data']['vlist']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
 
         elif sort == 'audio_menu':
             m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)

From 261874c018506d452c62f3ddb4ca49c17d1be753 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 28 Feb 2019 00:12:52 +0100
Subject: [PATCH 0813/1225] [tumblr] skip non-jpg

---
 src/you_get/extractors/tumblr.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 91b348fce3..61602197e7 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -49,7 +49,9 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         tuggles = {}
         for url in urls:
-            hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg'  # FIXME: decide actual quality
+            hd_url = r1(r'(.+)_\d+\.jpg$', url)  # FIXME: .png and .gif
+            if hd_url is None: continue
+            hd_url = hd_url + '_1280.jpg'  # FIXME: decide actual quality
             filename = parse.unquote(hd_url.split('/')[-1])
             title = '.'.join(filename.split('.')[:-1])
             tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)

From c19edfef350238ed8850cd5a941ebd160423b18c Mon Sep 17 00:00:00 2001
From: yawwwwwn <40122222+yawwwwwn@users.noreply.github.com>
Date: Fri, 1 Mar 2019 19:43:56 +0800
Subject: [PATCH 0814/1225] [pptv] support https url

---
 src/you_get/extractors/pptv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index 8d95a5a133..dacd78e4c7 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -190,7 +190,7 @@ class PPTV(VideoExtractor):
 
     def prepare(self, **kwargs):
         if self.url and not self.vid:
-            if not re.match(r'http://v.pptv.com/show/(\w+)\.html', self.url):
+            if not re.match(r'https?://v.pptv.com/show/(\w+)\.html', self.url):
                 raise('Unknown url pattern')
             page_content = get_content(self.url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"})
             self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)')

From d961f7f49e511316b670f443be22aad97ec71968 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 3 Mar 2019 17:30:48 +0100
Subject: [PATCH 0815/1225] [common] load_cookies(): load lines with prefix
 "#HttpOnly_" as MozillaCookieJar treats them as comments

---
 src/you_get/common.py | 62 +++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 60 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1d1088daa6..7719533216 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1276,10 +1276,68 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
 
 
 def load_cookies(cookiefile):
+    from http.cookiejar import Cookie
     global cookies
     try:
-        cookies = cookiejar.MozillaCookieJar(cookiefile)
-        cookies.load()
+        # MozillaCookieJar treats prefix '#HttpOnly_' as comments incorrectly!
+        # do not use its load()
+        # see also:
+        #   - https://docs.python.org/3/library/http.cookiejar.html#http.cookiejar.MozillaCookieJar
+        #   - https://github.com/python/cpython/blob/4b219ce/Lib/http/cookiejar.py#L2014
+        #   - https://curl.haxx.se/libcurl/c/CURLOPT_COOKIELIST.html#EXAMPLE
+        #cookies = cookiejar.MozillaCookieJar(cookiefile)
+        #cookies.load()
+        cookies = cookiejar.MozillaCookieJar()
+        now = time.time()
+        ignore_discard, ignore_expires = False, False
+        with open(cookiefile, 'r') as f:
+            for line in f:
+                # last field may be absent, so keep any trailing tab
+                if line.endswith("\n"): line = line[:-1]
+
+                # skip comments and blank lines XXX what is $ for?
+                if (line.strip().startswith(("#", "$")) or
+                    line.strip() == ""):
+                    if not line.strip().startswith('#HttpOnly_'):  # skip for #HttpOnly_
+                        continue
+
+                domain, domain_specified, path, secure, expires, name, value = \
+                        line.split("\t")
+                secure = (secure == "TRUE")
+                domain_specified = (domain_specified == "TRUE")
+                if name == "":
+                    # cookies.txt regards 'Set-Cookie: foo' as a cookie
+                    # with no name, whereas http.cookiejar regards it as a
+                    # cookie with no value.
+                    name = value
+                    value = None
+
+                initial_dot = domain.startswith(".")
+                if not line.strip().startswith('#HttpOnly_'):  # skip for #HttpOnly_
+                    assert domain_specified == initial_dot
+
+                discard = False
+                if expires == "":
+                    expires = None
+                    discard = True
+
+                # assume path_specified is false
+                c = Cookie(0, name, value,
+                           None, False,
+                           domain, domain_specified, initial_dot,
+                           path, False,
+                           secure,
+                           expires,
+                           discard,
+                           None,
+                           None,
+                           {})
+                if not ignore_discard and c.discard:
+                    continue
+                if not ignore_expires and c.is_expired(now):
+                    continue
+                cookies.set_cookie(c)
+
     except Exception:
         import sqlite3
         cookies = cookiejar.MozillaCookieJar()

From 37dd620fc1e1e6517955c91d24eb7a68b306f86b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 3 Mar 2019 17:40:54 +0100
Subject: [PATCH 0816/1225] [youtube] load necessary cookies into headers (for
 age-restricted videos)

---
 src/you_get/extractors/youtube.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 9c05e7874e..844c90eae4 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -195,7 +195,24 @@ def prepare(self, **kwargs):
 
         elif video_info['status'] == ['fail']:
             if video_info['errorcode'] == ['150']:
-                video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
+                if cookies:
+                    # Load necessary cookies into headers (for age-restricted videos)
+                    consent, ssid, hsid, sid = 'YES', '', '', ''
+                    for cookie in cookies:
+                        if cookie.domain.endswith('.youtube.com'):
+                            if cookie.name == 'SSID':
+                                ssid = cookie.value
+                            elif cookie.name == 'HSID':
+                                hsid = cookie.value
+                            elif cookie.name == 'SID':
+                                sid = cookie.value
+                    cookie_str = 'CONSENT=%s; SSID=%s; HSID=%s; SID=%s' % (consent, ssid, hsid, sid)
+
+                    video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid,
+                                             headers={'Cookie': cookie_str})
+                else:
+                    video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
+
                 try:
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+});ytplayer', video_page).group(1))
                 except:

From 2f9263c01fec71752582045e24f8064ba0395bb6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 3 Mar 2019 17:58:55 +0100
Subject: [PATCH 0817/1225] [youtube] hint for cookies (for age-restricted
 videos)

---
 src/you_get/extractors/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 844c90eae4..f33947c459 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -217,8 +217,7 @@ def prepare(self, **kwargs):
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+});ytplayer', video_page).group(1))
                 except:
                     msg = re.search('class="message">([^<]+)<', video_page).group(1)
-                    log.wtf('[Failed] "%s"' % msg.strip(), exit_code=None)
-                    raise
+                    log.wtf('[Failed] Got message "%s". Try to login with --cookies.' % msg.strip())
 
                 if 'title' in ytplayer_config['args']:
                     # 150 Restricted from playback on certain sites

From 077b604748c4054cbb956fcaa83e4cdf2635dcc6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 3 Mar 2019 18:28:58 +0100
Subject: [PATCH 0818/1225] [common] load_cookies(): copy cookies.sqlite to a
 temporary file in case database is locked (e.g., Firefox is running)

---
 src/you_get/common.py | 40 ++++++++++++++++++++++------------------
 1 file changed, 22 insertions(+), 18 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 7719533216..b95e7a52ee 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1276,9 +1276,8 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
 
 
 def load_cookies(cookiefile):
-    from http.cookiejar import Cookie
     global cookies
-    try:
+    if cookiefile.endswith('.txt'):
         # MozillaCookieJar treats prefix '#HttpOnly_' as comments incorrectly!
         # do not use its load()
         # see also:
@@ -1287,6 +1286,7 @@ def load_cookies(cookiefile):
         #   - https://curl.haxx.se/libcurl/c/CURLOPT_COOKIELIST.html#EXAMPLE
         #cookies = cookiejar.MozillaCookieJar(cookiefile)
         #cookies.load()
+        from http.cookiejar import Cookie
         cookies = cookiejar.MozillaCookieJar()
         now = time.time()
         ignore_discard, ignore_expires = False, False
@@ -1338,24 +1338,28 @@ def load_cookies(cookiefile):
                     continue
                 cookies.set_cookie(c)
 
-    except Exception:
-        import sqlite3
+    elif cookiefile.endswith(('.sqlite', '.sqlite3')):
+        import sqlite3, shutil, tempfile
+        temp_dir = tempfile.gettempdir()
+        temp_cookiefile = os.path.join(temp_dir, 'temp_cookiefile.sqlite')
+        shutil.copy2(cookiefile, temp_cookiefile)
+
         cookies = cookiejar.MozillaCookieJar()
-        con = sqlite3.connect(cookiefile)
+        con = sqlite3.connect(temp_cookiefile)
         cur = con.cursor()
-        try:
-            cur.execute("""SELECT host, path, isSecure, expiry, name, value
-                        FROM moz_cookies""")
-            for item in cur.fetchall():
-                c = cookiejar.Cookie(
-                    0, item[4], item[5], None, False, item[0],
-                    item[0].startswith('.'), item[0].startswith('.'),
-                    item[1], False, item[2], item[3], item[3] == '', None,
-                    None, {},
-                )
-                cookies.set_cookie(c)
-        except Exception:
-            pass
+        cur.execute("""SELECT host, path, isSecure, expiry, name, value
+        FROM moz_cookies""")
+        for item in cur.fetchall():
+            c = cookiejar.Cookie(
+                0, item[4], item[5], None, False, item[0],
+                item[0].startswith('.'), item[0].startswith('.'),
+                item[1], False, item[2], item[3], item[3] == '', None,
+                None, {},
+            )
+            cookies.set_cookie(c)
+
+    else:
+        log.e('[error] unsupported cookies format')
         # TODO: Chromium Cookies
         # SELECT host_key, path, secure, expires_utc, name, encrypted_value
         # FROM cookies

From 9cabacce649b9cb7f66f12b16f8050cf19fca90f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 5 Mar 2019 04:18:09 +0100
Subject: [PATCH 0819/1225] [universal] <img> with high widths

---
 src/you_get/extractors/universal.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 756ce4c1ca..69ef5d90fb 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -80,6 +80,9 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         urls += re.findall(r'href="(https?://[^"]+\.png)"', page, re.I)
         urls += re.findall(r'href="(https?://[^"]+\.gif)"', page, re.I)
 
+        # <img> with high widths
+        urls += re.findall(r'<img src="([^"]*)"[^>]*width="\d\d\d+"', page, re.I)
+
         # relative path
         rel_urls = []
         rel_urls += re.findall(r'href="(\.[^"]+\.jpe?g)"', page, re.I)
@@ -101,7 +104,7 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         for url in set(urls):
             filename = parse.unquote(url.split('/')[-1])
             if 5 <= len(filename) <= 80:
-                title = '.'.join(filename.split('.')[:-1])
+                title = '.'.join(filename.split('.')[:-1]) or filename
             else:
                 title = '%s' % i
                 i += 1

From 35f20be70c727c2adab8cf7e2e6ee5beb8ac1f29 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 7 Mar 2019 13:39:45 +0100
Subject: [PATCH 0820/1225] [youtube] format stream_types (Non-DASH YouTube
 media encoding options)

---
 src/you_get/extractors/youtube.py | 66 +++++++++++++++++++++++--------
 1 file changed, 49 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index f33947c459..552367dc2f 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -8,32 +8,64 @@
 class YouTube(VideoExtractor):
     name = "YouTube"
 
-    # YouTube media encoding options, in descending quality order.
+    # Non-DASH YouTube media encoding options, in descending quality order.
     # http://en.wikipedia.org/wiki/YouTube#Quality_and_codecs. Retrieved July 17, 2014.
     stream_types = [
-        {'itag': '38', 'container': 'MP4', 'video_resolution': '3072p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3.5-5', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+        {'itag': '38', 'container': 'MP4', 'video_resolution': '3072p',
+         'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3.5-5',
+         'audio_encoding': 'AAC', 'audio_bitrate': '192'},
         #{'itag': '85', 'container': 'MP4', 'video_resolution': '1080p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '3-4', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
-        {'itag': '46', 'container': 'WebM', 'video_resolution': '1080p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
-        {'itag': '37', 'container': 'MP4', 'video_resolution': '1080p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3-4.3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+        {'itag': '46', 'container': 'WebM', 'video_resolution': '1080p',
+         'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '',
+         'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+        {'itag': '37', 'container': 'MP4', 'video_resolution': '1080p',
+         'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '3-4.3',
+         'audio_encoding': 'AAC', 'audio_bitrate': '192'},
         #{'itag': '102', 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
-        {'itag': '45', 'container': 'WebM', 'video_resolution': '720p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '2', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
+        {'itag': '45', 'container': 'WebM', 'video_resolution': '720p',
+         'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '2',
+         'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
         #{'itag': '84', 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '2-3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
-        {'itag': '22', 'container': 'MP4', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '2-3', 'audio_encoding': 'AAC', 'audio_bitrate': '192'},
-        {'itag': '120', 'container': 'FLV', 'video_resolution': '720p', 'video_encoding': 'H.264', 'video_profile': 'Main@L3.1', 'video_bitrate': '2', 'audio_encoding': 'AAC', 'audio_bitrate': '128'}, # Live streaming only
-        {'itag': '44', 'container': 'WebM', 'video_resolution': '480p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '1', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
-        {'itag': '35', 'container': 'FLV', 'video_resolution': '480p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.8-1', 'audio_encoding': 'AAC', 'audio_bitrate': '128'},
+        {'itag': '22', 'container': 'MP4', 'video_resolution': '720p',
+         'video_encoding': 'H.264', 'video_profile': 'High', 'video_bitrate': '2-3',
+         'audio_encoding': 'AAC', 'audio_bitrate': '192'},
+        {'itag': '120', 'container': 'FLV', 'video_resolution': '720p',
+         'video_encoding': 'H.264', 'video_profile': 'Main@L3.1', 'video_bitrate': '2',
+         'audio_encoding': 'AAC', 'audio_bitrate': '128'}, # Live streaming only
+        {'itag': '44', 'container': 'WebM', 'video_resolution': '480p',
+         'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '1',
+         'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
+        {'itag': '35', 'container': 'FLV', 'video_resolution': '480p',
+         'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.8-1',
+         'audio_encoding': 'AAC', 'audio_bitrate': '128'},
         #{'itag': '101', 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '192'},
         #{'itag': '100', 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '3D', 'video_bitrate': '', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
-        {'itag': '43', 'container': 'WebM', 'video_resolution': '360p', 'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
-        {'itag': '34', 'container': 'FLV', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '128'},
+        {'itag': '43', 'container': 'WebM', 'video_resolution': '360p',
+         'video_encoding': 'VP8', 'video_profile': '', 'video_bitrate': '0.5',
+         'audio_encoding': 'Vorbis', 'audio_bitrate': '128'},
+        {'itag': '34', 'container': 'FLV', 'video_resolution': '360p',
+         'video_encoding': 'H.264', 'video_profile': 'Main', 'video_bitrate': '0.5',
+         'audio_encoding': 'AAC', 'audio_bitrate': '128'},
         #{'itag': '82', 'container': 'MP4', 'video_resolution': '360p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
-        {'itag': '18', 'container': 'MP4', 'video_resolution': '270p/360p', 'video_encoding': 'H.264', 'video_profile': 'Baseline', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
-        {'itag': '6', 'container': 'FLV', 'video_resolution': '270p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.8', 'audio_encoding': 'MP3', 'audio_bitrate': '64'},
+        {'itag': '18', 'container': 'MP4', 'video_resolution': '360p',
+         'video_encoding': 'H.264', 'video_profile': 'Baseline', 'video_bitrate': '0.5',
+         'audio_encoding': 'AAC', 'audio_bitrate': '96'},
+        {'itag': '6', 'container': 'FLV', 'video_resolution': '270p',
+         'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.8',
+         'audio_encoding': 'MP3', 'audio_bitrate': '64'},
         #{'itag': '83', 'container': 'MP4', 'video_resolution': '240p', 'video_encoding': 'H.264', 'video_profile': '3D', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': '96'},
-        {'itag': '13', 'container': '3GP', 'video_resolution': '', 'video_encoding': 'MPEG-4 Visual', 'video_profile': '', 'video_bitrate': '0.5', 'audio_encoding': 'AAC', 'audio_bitrate': ''},
-        {'itag': '5', 'container': 'FLV', 'video_resolution': '240p', 'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.25', 'audio_encoding': 'MP3', 'audio_bitrate': '64'},
-        {'itag': '36', 'container': '3GP', 'video_resolution': '240p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.175', 'audio_encoding': 'AAC', 'audio_bitrate': '36'},
-        {'itag': '17', 'container': '3GP', 'video_resolution': '144p', 'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.05', 'audio_encoding': 'AAC', 'audio_bitrate': '24'},
+        {'itag': '13', 'container': '3GP', 'video_resolution': '',
+         'video_encoding': 'MPEG-4 Visual', 'video_profile': '', 'video_bitrate': '0.5',
+         'audio_encoding': 'AAC', 'audio_bitrate': ''},
+        {'itag': '5', 'container': 'FLV', 'video_resolution': '240p',
+         'video_encoding': 'Sorenson H.263', 'video_profile': '', 'video_bitrate': '0.25',
+         'audio_encoding': 'MP3', 'audio_bitrate': '64'},
+        {'itag': '36', 'container': '3GP', 'video_resolution': '240p',
+         'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.175',
+         'audio_encoding': 'AAC', 'audio_bitrate': '32'},
+        {'itag': '17', 'container': '3GP', 'video_resolution': '144p',
+         'video_encoding': 'MPEG-4 Visual', 'video_profile': 'Simple', 'video_bitrate': '0.05',
+         'audio_encoding': 'AAC', 'audio_bitrate': '24'},
     ]
 
     def decipher(js, s):

From dca9897b8acdabbbe7e1638f4ff09bd2861f1221 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 7 Mar 2019 13:47:31 +0100
Subject: [PATCH 0821/1225] [youtube] show quality_label

---
 src/you_get/extractors/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 552367dc2f..abef0403df 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -437,7 +437,7 @@ def prepare(self, **kwargs):
                             dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
                             dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
                             self.dash_streams[itag] = {
-                                'quality': stream['size'],
+                                'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
                                 'itag': itag,
                                 'type': mimeType,
                                 'mime': mimeType,
@@ -464,7 +464,7 @@ def prepare(self, **kwargs):
                             dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
                             audio_urls = self.__class__.chunk_by_range(audio_url, int(audio_size))
                             self.dash_streams[itag] = {
-                                'quality': stream['size'],
+                                'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
                                 'itag': itag,
                                 'type': mimeType,
                                 'mime': mimeType,

From 6563bd60c0f7fb1acf2dd84c3e3a616fdded7947 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Mar 2019 23:17:28 +0100
Subject: [PATCH 0822/1225] update CONTRIBUTING.md

---
 .github/ISSUE_TEMPLATE.md        | 39 --------------------------
 .github/PULL_REQUEST_TEMPLATE.md | 48 --------------------------------
 CONTRIBUTING.md                  | 32 ++++++++++-----------
 README.md                        |  6 +++-
 4 files changed, 21 insertions(+), 104 deletions(-)
 delete mode 100644 .github/ISSUE_TEMPLATE.md
 delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md

diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
deleted file mode 100644
index 8556750719..0000000000
--- a/.github/ISSUE_TEMPLATE.md
+++ /dev/null
@@ -1,39 +0,0 @@
-Please make sure these boxes are checked before submitting your issue – thank you!
-
-- [ ] You can actually watch the video in your browser or mobile application, but not download them with `you-get`.
-- [ ] Your `you-get` is up-to-date.
-- [ ] I have read <https://github.com/soimort/you-get/wiki/FAQ> and tried to do so.
-- [ ] The issue is not yet reported on <https://github.com/soimort/you-get/issues> or  <https://github.com/soimort/you-get/wiki/Known-Bugs>. If so, please add your comments under the existing issue.
-- [ ] The issue (or question) is really about `you-get`, not about some other code or project.
-
-Run the command with the `--debug` option, and paste the full output inside the fences:
-
-```
-[PASTE IN ME]
-```
-
-If there's anything else you would like to say (e.g. in case your issue is not about downloading a specific video; it might as well be a general discussion or proposal for a new feature), fill in the box below; otherwise, you may want to post an emoji or meme instead:
-
-> [WRITE SOMETHING]
-> [OR HAVE SOME :icecream:!]
-
-汉语翻译最终日期：2016年02月26日
-
-在提交前，请确保您已经检查了以下内容!
-
-- [ ] 你可以在浏览器或移动端中观看视频，但不能使用`you-get`下载.
-- [ ] 您的`you-get`为最新版.
-- [ ] 我已经阅读并按 <https://github.com/soimort/you-get/wiki/FAQ> 中的指引进行了操作.
-- [ ] 您的问题没有在<https://github.com/soimort/you-get/issues> , <https://github.com/soimort/you-get/wiki/FAQ> 或 <https://github.com/soimort/you-get/wiki/Known-Bugs> 报告，否则请在原有issue下报告.
-- [ ] 本问题确实关于`you-get`, 而不是其他项目.
-
-请使用`--debug`运行，并将输出粘贴在下面:
-
-```
-[在这里粘贴完整日志]
-```
-
-如果您有其他附言，例如问题只在某个视频发生，或者是一般性讨论或者提出新功能，请在下面添加;或者您可以卖个萌:
-
-> [您的内容]
-> [舔 :icecream:!]
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
deleted file mode 100644
index 79a43f6bc4..0000000000
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ /dev/null
@@ -1,48 +0,0 @@
-**(PLEASE DELETE ALL THESE AFTER READING)**
-
-Thank you for the pull request! `you-get` is a growing open source project, which would not have been possible without contributors like you.
-
-Here are some simple rules to follow, please recheck them before sending the pull request:
-
-- [ ] If you want to propose two or more unrelated patches, please open separate pull requests for them, instead of one;
-- [ ] All pull requests should be based upon the latest `develop` branch;
-- [ ] Name your branch (from which you will send the pull request) properly; use a meaningful name like `add-this-shining-feature` rather than just `develop`;
-- [ ] All commit messages, as well as comments in code, should be written in understandable English.
-
-As a contributor, you must be aware that
-
-- [ ] You agree to contribute your code to this project, under the terms of the MIT license, so that any person may freely use or redistribute them; of course, you will still reserve the copyright for your own authorship.
-- [ ] You may not contribute any code not authored by yourself, unless they are licensed under either public domain or the MIT license, literally.
-
-Not all pull requests can eventually be merged. I consider merged / unmerged patches as equally important for the community: as long as you think a patch would be helpful, someone else might find it helpful, too, therefore they could take your fork and benefit in some way. In any case, I would like to thank you in advance for taking your time to contribute to this project.
-
-Cheers,
-Mort
-
-**(PLEASE REPLACE ALL ABOVE WITH A DETAILED DESCRIPTION OF YOUR PULL REQUEST)**
-
-
-汉语翻译最后日期：2016年02月26日
-
-**(阅读后请删除所有内容)**
-
-感谢您的pull request! `you-get`是稳健成长的开源项目，感谢您的贡献.
-
-以下简单检查项目望您复查:
-
-- [ ] 如果您预计提出两个或更多不相关补丁，请为每个使用不同的pull requests，而不是单一;
-- [ ] 所有的pull requests应基于最新的`develop`分支;
-- [ ] 您预计提出pull requests的分支应有有意义名称，例如`add-this-shining-feature`而不是`develop`;
-- [ ] 所有的提交信息与代码中注释应使用可理解的英语.
-
-作为贡献者，您需要知悉
-
-- [ ] 您同意在MIT协议下贡献代码，以便任何人自由使用或分发;当然，你仍旧保留代码的著作权
-- [ ] 你不得贡献非自己编写的代码，除非其属于公有领域或使用MIT协议.
-
-不是所有的pull requests都会被合并,然而我认为合并/不合并的补丁一样重要：如果您认为补丁重要，其他人也有可能这么认为，那么他们可以从你的fork中提取工作并获益。无论如何，感谢您费心对本项目贡献.
-
-祝好,
-Mort
-
-**(请将本内容完整替换为PULL REQUEST的详细内容)**
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b7b6ba42b6..368169480d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,27 +1,27 @@
-# How to Contribute
+# How to Report an Issue
 
-`you-get` is currently experimenting with an aggressive approach to handling issues. Namely, a bug report must be addressed with some code via a pull request.
+If you would like to report a problem you find when using `you-get`, please open a [Pull Request](https://github.com/soimort/you-get/pulls), which should include:
 
-## Report a broken extractor
+1. A detailed description of the encountered problem;
+2. At least one commit, addressing the problem through some unit test(s).
+   * Examples of good commits: [#2675](https://github.com/soimort/you-get/pull/2675/files), [#2680](https://github.com/soimort/you-get/pull/2680/files), [#2685](https://github.com/soimort/you-get/pull/2685/files)
 
-**How-To:** Please open a new pull request with the following changes:
+PRs that fail to meet the above criteria may be closed summarily with no further action.
 
-* Add a new test case in [tests/test.py](https://github.com/soimort/you-get/blob/develop/tests/test.py), with the failing URL(s).
+A valid PR will remain open until its addressed problem is fixed.
 
-The Travis CI build will (ideally) fail showing a :x:, which means you have successfully reported a broken extractor.
 
-Such a valid PR will be either *closed* if it's fixed by another PR, or *merged* if it's fixed by follow-up commits from the reporter himself/herself.
 
-## Report other issues / Suggest a new feature
+# 如何汇报问题
 
-**How-To:** Please open a pull request with the proposed changes directly.
+为了防止对 GitHub Issues 的滥用，本项目不接受一般的 Issue。
 
-A valid PR need not be complete (i.e., can be WIP), but it should contain at least one sensible, nontrivial commit.
+如您在使用 `you-get` 的过程中发现任何问题，请开启一个 [Pull Request](https://github.com/soimort/you-get/pulls)。该 PR 应当包含：
 
-## Hints
+1. 详细的问题描述；
+2. 至少一个 commit，其内容是**与问题相关的**单元测试。**不要通过随意修改无关文件的方式来提交 PR！**
+   * 有效的 commit 示例：[#2675](https://github.com/soimort/you-get/pull/2675/files), [#2680](https://github.com/soimort/you-get/pull/2680/files), [#2685](https://github.com/soimort/you-get/pull/2685/files)
 
-* The [`develop`](https://github.com/soimort/you-get/tree/develop) branch is where your pull request goes.
-* Remember to rebase.
-* Document your PR clearly, and if applicable, provide some sample links for reviewers to test with.
-* Write well-formatted, easy-to-understand commit messages. If you don't know how, look at existing ones.
-* We will not ask you to sign a CLA, but you must assure that your code can be legally redistributed (under the terms of the MIT license).
+不符合以上条件的 PR 可能被直接关闭。
+
+有效的 PR 将会被一直保留，直至相应的问题得以修复。
diff --git a/README.md b/README.md
index 6a9746aeee..e1551c9a54 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,10 @@
 [![Build Status](https://travis-ci.org/soimort/you-get.svg)](https://travis-ci.org/soimort/you-get)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
+**NOTICE: Read [this](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) if you are looking for the conventional "Issues" tab.**
+
+---
+
 [You-Get](https://you-get.org/) is a tiny command-line utility to download media contents (videos, audios, images) from the Web, in case there is no other handy way to do it.
 
 Here's how you use `you-get` to download a video from [YouTube](https://www.youtube.com/watch?v=jNQXAC9IVRw):
@@ -448,7 +452,7 @@ For all other sites not on the list, the universal extractor will take care of f
 
 If something is broken and `you-get` can't get you things you want, don't panic. (Yes, this happens all the time!)
 
-Check if it's already a known problem on <https://github.com/soimort/you-get/wiki/Known-Bugs>. If not, follow the guidelines on [how to report a broken extractor](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md#report-a-broken-extractor).
+Check if it's already a known problem on <https://github.com/soimort/you-get/wiki/Known-Bugs>. If not, follow the guidelines on [how to report an issue](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md).
 
 ## Getting Involved
 

From 78e392cdca0023a1f15520066aaf36e116f62057 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Mar 2019 23:18:41 +0100
Subject: [PATCH 0823/1225] [google] fix for Google+ photos

---
 src/you_get/extractors/google.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/google.py b/src/you_get/extractors/google.py
index bf64b24324..c3de296ee4 100644
--- a/src/you_get/extractors/google.py
+++ b/src/you_get/extractors/google.py
@@ -59,7 +59,7 @@ def google_download(url, output_dir = '.', merge = True, info_only = False, **kw
             u = '/'.join(t)
             real_urls.append(u)
         if not real_urls:
-            real_urls = [r1(r'<meta property="og:image" content="([^"]+)', html)]
+            real_urls = re.findall(r'<meta property="og:image" content="([^"]+)', html)
             real_urls = [re.sub(r'w\d+-h\d+-p', 's0', u) for u in real_urls]
         post_date = r1(r'"?(20\d\d[-/]?[01]\d[-/]?[0123]\d)"?', html)
         post_id = r1(r'/posts/([^"]+)', html)

From 3ef69d87b2ca39392141061dbd54962c41f4cec1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Mar 2019 23:20:06 +0100
Subject: [PATCH 0824/1225] version 0.4.1270

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 663937491d..d3c39cd09d 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1256'
+__version__ = '0.4.1270'

From bc3460b3e3d45e21f404e3b51c8da474cb42516b Mon Sep 17 00:00:00 2001
From: "edward.gao" <edward.gao@logicmonitor.com>
Date: Tue, 12 Mar 2019 13:40:02 +0800
Subject: [PATCH 0825/1225] Support ignore ssl when connecting through vpn by
 adding option -k

---
 src/you_get/common.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index b19d602f14..a3a98212b8 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -10,6 +10,7 @@
 import locale
 import logging
 import argparse
+import ssl
 from http import cookiejar
 from importlib import import_module
 from urllib import request, parse, error
@@ -137,6 +138,7 @@
 cookies = None
 output_filename = None
 auto_rename = False
+in_secure = False
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
@@ -385,7 +387,13 @@ def urlopen_with_retry(*args, **kwargs):
     retry_time = 3
     for i in range(retry_time):
         try:
-            return request.urlopen(*args, **kwargs)
+            if in_secure:
+                ctx = ssl.create_default_context()
+                ctx.check_hostname = False
+                ctx.verify_mode = ssl.CERT_NONE
+                return request.urlopen(*args, **kwargs, context=ctx)
+            else:
+                return request.urlopen(*args, **kwargs)
         except socket.timeout as e:
             logging.debug('request attempt %s timeout' % str(i + 1))
             if i + 1 == retry_time:
@@ -1397,6 +1405,11 @@ def print_version():
         help='Auto rename same name different files'
     )
 
+    download_grp.add_argument(
+        '-k', '--in-secure', action='store_true', default=False,
+        help='ignore ssl errors'
+    )
+
     proxy_grp = parser.add_argument_group('Proxy options')
     proxy_grp = proxy_grp.add_mutually_exclusive_group()
     proxy_grp.add_argument(
@@ -1441,7 +1454,7 @@ def print_version():
     global extractor_proxy
     global output_filename
     global auto_rename
-
+    global in_secure
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
@@ -1469,6 +1482,11 @@ def print_version():
         player = args.player
         caption = False
 
+    if args.in_secure:
+        # ignore ssl
+        in_secure = True
+
+
     if args.no_proxy:
         set_http_proxy('')
     else:

From e5f02725ea2098ca543f511ac696363e3bc20539 Mon Sep 17 00:00:00 2001
From: "edward.gao" <edward.gao@logicmonitor.com>
Date: Tue, 12 Mar 2019 13:58:13 +0800
Subject: [PATCH 0826/1225] Support ignore ssl when connecting through vpn by
 adding option -k

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index a3a98212b8..6e5b3be36e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -391,7 +391,7 @@ def urlopen_with_retry(*args, **kwargs):
                 ctx = ssl.create_default_context()
                 ctx.check_hostname = False
                 ctx.verify_mode = ssl.CERT_NONE
-                return request.urlopen(*args, **kwargs, context=ctx)
+                return request.urlopen(*args, context=ctx, **kwargs)
             else:
                 return request.urlopen(*args, **kwargs)
         except socket.timeout as e:

From 3e40e4bcaac06fd052ca26d54c9f5637b3792d1a Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Sat, 16 Mar 2019 05:09:40 +0800
Subject: [PATCH 0827/1225] fit "m.facebook.com" and so on

update
---
 src/you_get/extractors/facebook.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index 7134c56cb6..ba8aea8c72 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -6,6 +6,7 @@
 import json
 
 def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    url = re.sub(r'//.*?facebook.com','//facebook.com',url)
     html = get_html(url)
 
     title = r1(r'<title id="pageTitle">(.+)</title>', html)

From bbf3e8a3e28ee377a809adf0eb23a9a3bd263878 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 17 Mar 2019 22:56:08 +0100
Subject: [PATCH 0828/1225] [tumblr] glad to see people still use tumblr these
 days

---
 src/you_get/extractors/tumblr.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 61602197e7..9a314c7f03 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -49,9 +49,14 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         tuggles = {}
         for url in urls:
-            hd_url = r1(r'(.+)_\d+\.jpg$', url)  # FIXME: .png and .gif
-            if hd_url is None: continue
-            hd_url = hd_url + '_1280.jpg'  # FIXME: decide actual quality
+            if url.endswith('.gif'):
+                hd_url = url
+            elif url.endswith('.jpg'):
+                hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg' # FIXME: decide actual quality
+            elif url.endswith('.png'):
+                hd_url = r1(r'(.+)_\d+\.png$', url) + '_1280.png' # FIXME: decide actual quality
+            else:
+                continue
             filename = parse.unquote(hd_url.split('/')[-1])
             title = '.'.join(filename.split('.')[:-1])
             tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)

From a26b6ccc974489a6e57541c0ef520625af3a109a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Mar 2019 00:01:20 +0100
Subject: [PATCH 0829/1225] [embed] pass around **kwargs

---
 src/you_get/extractors/baidu.py |  3 ++-
 src/you_get/extractors/embed.py | 18 +++++++++---------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index a8cb3d5d74..f915afc970 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -123,7 +123,8 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
     elif re.match('http://tieba.baidu.com/', url):
         try:
             # embedded videos
-            embed_download(url, output_dir, merge=merge, info_only=info_only)
+            print(kwargs)
+            embed_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
         except:
             # images
             html = get_html(url)
diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index fa05181fd6..035a050b22 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -67,7 +67,7 @@
 recur_limit = 3
 
 
-def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwargs):
+def embed_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     content = get_content(url, headers=fake_headers)
     found = False
     title = match1(content, '<title>([^<>]+)</title>')
@@ -75,43 +75,43 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False ,**kwa
     vids = matchall(content, youku_embed_patterns)
     for vid in set(vids):
         found = True
-        youku_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        youku_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     vids = matchall(content, tudou_embed_patterns)
     for vid in set(vids):
         found = True
-        tudou_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        tudou_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     vids = matchall(content, yinyuetai_embed_patterns)
     for vid in vids:
         found = True
-        yinyuetai_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        yinyuetai_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     vids = matchall(content, iqiyi_embed_patterns)
     for vid in vids:
         found = True
-        iqiyi_download_by_vid((vid[1], vid[0]), title=title, output_dir=output_dir, merge=merge, info_only=info_only)
+        iqiyi_download_by_vid((vid[1], vid[0]), title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     urls = matchall(content, netease_embed_patterns)
     for url in urls:
         found = True
-        netease_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
+        netease_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     urls = matchall(content, vimeo_embed_patters)
     for url in urls:
         found = True
-        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url)
+        vimeo_download_by_id(url, title=title, output_dir=output_dir, merge=merge, info_only=info_only, referer=url, **kwargs)
 
     urls = matchall(content, dailymotion_embed_patterns)
     for url in urls:
         found = True
-        dailymotion_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
+        dailymotion_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     aids = matchall(content, bilibili_embed_patterns)
     for aid in aids:
         found = True
         url = 'http://www.bilibili.com/video/av%s/' % aid
-        bilibili_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
+        bilibili_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
     iqiyi_urls = matchall(content, iqiyi_patterns)
     for url in iqiyi_urls:

From beeb9d869225473de19c89aee13437c9aeeb44fc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 23 Mar 2019 00:05:42 +0100
Subject: [PATCH 0830/1225] [baidu] oops

---
 src/you_get/extractors/baidu.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index f915afc970..d3744f9dff 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -123,7 +123,6 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
     elif re.match('http://tieba.baidu.com/', url):
         try:
             # embedded videos
-            print(kwargs)
             embed_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
         except:
             # images

From 18e71ed9d91f3bbbed50dca76d21f62332c37375 Mon Sep 17 00:00:00 2001
From: "edward.gao" <edward.gao@logicmonitor.com>
Date: Mon, 25 Mar 2019 13:36:47 +0800
Subject: [PATCH 0831/1225] change the name to insecure instead of in-secure

---
 src/you_get/common.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 6e5b3be36e..4031ce3c7e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -138,7 +138,7 @@
 cookies = None
 output_filename = None
 auto_rename = False
-in_secure = False
+insecure = False
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
@@ -387,7 +387,8 @@ def urlopen_with_retry(*args, **kwargs):
     retry_time = 3
     for i in range(retry_time):
         try:
-            if in_secure:
+            if insecure:
+                # ignore ssl errors
                 ctx = ssl.create_default_context()
                 ctx.check_hostname = False
                 ctx.verify_mode = ssl.CERT_NONE
@@ -1406,7 +1407,7 @@ def print_version():
     )
 
     download_grp.add_argument(
-        '-k', '--in-secure', action='store_true', default=False,
+        '-k', '--insecure', action='store_true', default=False,
         help='ignore ssl errors'
     )
 
@@ -1454,7 +1455,7 @@ def print_version():
     global extractor_proxy
     global output_filename
     global auto_rename
-    global in_secure
+    global insecure
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
@@ -1482,9 +1483,9 @@ def print_version():
         player = args.player
         caption = False
 
-    if args.in_secure:
+    if args.insecure:
         # ignore ssl
-        in_secure = True
+        insecure = True
 
 
     if args.no_proxy:

From 10cc42f1fb9cbff01df4cc14b7be58fe90615f95 Mon Sep 17 00:00:00 2001
From: jason <jasonqiao36@gmail.com>
Date: Fri, 29 Mar 2019 22:01:08 +0800
Subject: [PATCH 0832/1225] fix toutiao errors

---
 src/you_get/extractors/toutiao.py | 45 ++++++++++++++++++-------------
 tests/test.py                     |  5 ++++
 2 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/src/you_get/extractors/toutiao.py b/src/you_get/extractors/toutiao.py
index 03f7a13b6a..1c356055f0 100644
--- a/src/you_get/extractors/toutiao.py
+++ b/src/you_get/extractors/toutiao.py
@@ -1,27 +1,36 @@
 #!/usr/bin/env python
-import base64
-
 import binascii
-
-from ..common import *
 import random
 from json import loads
+from urllib.parse import urlparse
+
+from ..common import *
+
+try:
+    from base64 import decodebytes
+except ImportError:
+    from base64 import decodestring
+
+    decodebytes = decodestring
 
 __all__ = ['toutiao_download', ]
 
 
-def sign_video_url(vid):
-    # some code from http://codecloud.net/110854.html
-    r = str(random.random())[2:]
+def random_with_n_digits(n):
+    return random.randint(10 ** (n - 1), (10 ** n) - 1)
 
-    def right_shift(val, n):
-        return val >> n if val >= 0 else (val + 0x100000000) >> n
 
-    url = 'http://i.snssdk.com/video/urls/v/1/toutiao/mp4/%s' % vid
-    n = url.replace("http://i.snssdk.com", "")+ '?r=' + r
-    c = binascii.crc32(n.encode("ascii"))
-    s = right_shift(c, 0)
-    return url + '?r=%s&s=%s' % (r, s)
+def sign_video_url(vid):
+    r = str(random_with_n_digits(16))
+
+    url = 'https://ib.365yg.com/video/urls/v/1/toutiao/mp4/{vid}'.format(vid=vid)
+    n = urlparse(url).path + '?r=' + r
+    b_n = bytes(n, encoding="utf-8")
+    s = binascii.crc32(b_n)
+    aid = 1364
+    ts = int(time.time() * 1000)
+    return url + '?r={r}&s={s}&aid={aid}&vfrom=xgplayer&callback=axiosJsonpCallback1&_={ts}'.format(r=r, s=s, aid=aid,
+                                                                                                    ts=ts)
 
 
 class ToutiaoVideoInfo(object):
@@ -43,12 +52,12 @@ def get_file_by_vid(video_id):
     vRet = []
     url = sign_video_url(video_id)
     ret = get_content(url)
-    ret = loads(ret)
+    ret = loads(ret[20:-1])
     vlist = ret.get('data').get('video_list')
     if len(vlist) > 0:
         vInfo = vlist.get(sorted(vlist.keys(), reverse=True)[0])
         vUrl = vInfo.get('main_url')
-        vUrl = base64.decodestring(vUrl.encode('ascii')).decode('ascii')
+        vUrl = decodebytes(vUrl.encode('ascii')).decode('ascii')
         videoInfo = ToutiaoVideoInfo()
         videoInfo.bitrate = vInfo.get('bitrate')
         videoInfo.definition = vInfo.get('definition')
@@ -63,8 +72,8 @@ def get_file_by_vid(video_id):
 
 def toutiao_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url, faker=True)
-    video_id = match1(html, r"videoid\s*:\s*'([^']+)',\n")
-    title = match1(html, r"title: '([^']+)'.replace")
+    video_id = match1(html, r".*?videoId: '(?P<vid>.*)'")
+    title = match1(html, '.*?<title>(?P<title>.*?)</title>')
     video_file_list = get_file_by_vid(video_id)  # 调api获取视频源文件
     type, ext, size = url_info(video_file_list[0].url, faker=True)
     print_info(site_info=site_info, title=title, type=type, size=size)
diff --git a/tests/test.py b/tests/test.py
index 047cdb0fa0..20b8de5050 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -7,6 +7,7 @@
     magisto,
     youtube,
     bilibili,
+    toutiao,
 )
 
 
@@ -31,5 +32,9 @@ def test_youtube(self):
             info_only=True
         )
 
+    def test_toutiao(self):
+        toutiao.download('https://www.365yg.com/i6640053613567675662/#mid=1611922564114440', info_only=True)
+
+
 if __name__ == '__main__':
     unittest.main()

From bc74c8e0ca24aeca9c34ad900f76b0f24c0bdc78 Mon Sep 17 00:00:00 2001
From: URenko <18209292+URenko@users.noreply.github.com>
Date: Mon, 15 Apr 2019 15:13:26 +0800
Subject: [PATCH 0833/1225] fix acfun bangumi title

---
 src/you_get/extractors/acfun.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 200a3f542b..424418adb8 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -121,7 +121,7 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # bangumi
     elif re.match("http://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url)
-        title = match1(html, r'"newTitle"\s*:\s*"([^"]+)"')
+        title = match1(html, r'"title"\s*:\s*"([^"]+)"')
         if match1(url, r'_(\d+)$'):  # current P
             title = title + " " + r1(r'active">([^<]*)', html)
         vid = match1(html, r'videoId="(\d+)"')

From e58eea43004d9df6b5b1e1f583e9fd17c11e93e8 Mon Sep 17 00:00:00 2001
From: dbw9580 <dbw9580@live.com>
Date: Wed, 17 Apr 2019 15:24:10 +0800
Subject: [PATCH 0834/1225] Replace broken api to get correct video title

---
 src/you_get/extractors/iqiyi.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 8e93959acd..3fe932091e 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -136,12 +136,9 @@ def prepare(self, **kwargs):
                       r1(r'vid=([^&]+)', self.url) or \
                       r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(.+?)\&', html) or r1(r'param\[\'vid\'\]\s*=\s*"(.+?)"', html)
             self.vid = (tvid, videoid)
-            info_u = 'http://mixer.video.iqiyi.com/jp/mixin/videos/' + tvid
-            mixin = get_content(info_u)
-            mixin_json = json.loads(mixin[len('var tvInfoJs='):])
-            real_u = mixin_json['url']
-            real_html = get_content(real_u)
-            self.title = match1(real_html, '<title>([^<]+)').split('-')[0]
+            info_u = 'http://pcw-api.iqiyi.com/video/video/playervideoinfo?tvid=' + tvid
+            json_res = get_content(info_u)
+            self.title = json.loads(json_res)['data']['vn']
         tvid, videoid = self.vid
         info = getVMS(tvid, videoid)
         assert info['code'] == 'A00000', "can't play this video"

From 1658fd32fcaeb168d68326165dfd46b2b53c8e96 Mon Sep 17 00:00:00 2001
From: icemoon1987 <panwenhai1987@163.com>
Date: Thu, 18 Apr 2019 16:50:13 +0800
Subject: [PATCH 0835/1225] add --skip-ignore-size function

---
 src/you_get/common.py | 33 ++++++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 87c31f8975..bb9ec4a95d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -131,6 +131,7 @@
 dry_run = False
 json_output = False
 force = False
+skip_ignore_size = False
 player = None
 extractor_proxy = None
 cookies = None
@@ -633,15 +634,22 @@ def url_save(
     while continue_renameing:
         continue_renameing = False
         if os.path.exists(filepath):
-            if not force and file_size == os.path.getsize(filepath):
+            if not force and (file_size == os.path.getsize(filepath) or skip_ignore_size):
                 if not is_part:
                     if bar:
                         bar.done()
-                    log.w(
-                        'Skipping {}: file already exists'.format(
-                            tr(os.path.basename(filepath))
+                    if skip_ignore_size:
+                        log.w(
+                            'Skipping {} without checking size: file already exists'.format(
+                                tr(os.path.basename(filepath))
+                            )
+                        )
+                    else:
+                        log.w(
+                            'Skipping {}: file already exists'.format(
+                                tr(os.path.basename(filepath))
+                            )
                         )
-                    )
                 else:
                     if bar:
                         bar.update_received(file_size)
@@ -945,8 +953,12 @@ def download_urls(
 
     if total_size:
         if not force and os.path.exists(output_filepath) and not auto_rename\
-                and os.path.getsize(output_filepath) >= total_size * 0.9:
-            log.w('Skipping %s: file already exists' % output_filepath)
+                and (os.path.getsize(output_filepath) >= total_size * 0.9\
+                or skip_ignore_size):
+            if skip_ignore_size:
+                log.w('Skipping %s without checking size: file already exists' % output_filepath)
+            else:
+                log.w('Skipping %s: file already exists' % output_filepath)
             print()
             return
         bar = SimpleProgressBar(total_size, len(urls))
@@ -1455,6 +1467,10 @@ def print_version():
         '-f', '--force', action='store_true', default=False,
         help='Force overwriting existing files'
     )
+    download_grp.add_argument(
+        '--skip-ignore-size', action='store_true', default=False,
+        help='Skip existing file without checking file size'
+    )
     download_grp.add_argument(
         '-F', '--format', metavar='STREAM_ID',
         help='Set video format to STREAM_ID'
@@ -1541,6 +1557,7 @@ def print_version():
         logging.getLogger().setLevel(logging.DEBUG)
 
     global force
+    global skip_ignore_size
     global dry_run
     global json_output
     global player
@@ -1554,6 +1571,8 @@ def print_version():
     info_only = args.info
     if args.force:
         force = True
+    if args.skip_ignore_size:
+        skip_ignore_size = True
     if args.auto_rename:
         auto_rename = True
     if args.url:

From 7ce9371aba2cdbb523f59190db17986d6ac208a8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Apr 2019 00:47:58 +0200
Subject: [PATCH 0836/1225] [youtube] bypass age restriction (again)

---
 src/you_get/extractors/youtube.py | 143 +++++++++++++++++-------------
 1 file changed, 80 insertions(+), 63 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index abef0403df..0fcf2b6974 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -195,7 +195,11 @@ def prepare(self, **kwargs):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
-        video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}'.format(self.vid)))
+        # Get video info
+        # 'eurl' is a magic parameter that can bypass age restriction
+        # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
+        video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
+        logging.debug('STATUS: %s' % video_info['status'][0])
 
         ytplayer_config = None
         if 'status' not in video_info:
@@ -214,7 +218,10 @@ def prepare(self, **kwargs):
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                 except:
                     stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
-                    self.html5player = None
+                    if re.search('([^"]*/base\.js)"', video_page):
+                        self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                    else:
+                        self.html5player = None
 
             else:
                 # Parse video page instead
@@ -226,7 +233,9 @@ def prepare(self, **kwargs):
                 stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
 
         elif video_info['status'] == ['fail']:
+            logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])
             if video_info['errorcode'] == ['150']:
+                # FIXME: still relevant?
                 if cookies:
                     # Load necessary cookies into headers (for age-restricted videos)
                     consent, ssid, hsid, sid = 'YES', '', '', ''
@@ -404,74 +413,82 @@ def prepare(self, **kwargs):
             # VEVO
             if not self.html5player: return
             self.js = get_content(self.html5player)
-            if 'adaptive_fmts' in ytplayer_config['args']:
+
+            try:
+                # Video info from video page (not always available)
+                streams = [dict([(i.split('=')[0],
+                                  parse.unquote(i.split('=')[1]))
+                                 for i in afmt.split('&')])
+                           for afmt in ytplayer_config['args']['adaptive_fmts'][0].split(',')]
+            except:
                 streams = [dict([(i.split('=')[0],
                                   parse.unquote(i.split('=')[1]))
                                  for i in afmt.split('&')])
-                           for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
-                for stream in streams: # get over speed limiting
-                    stream['url'] += '&ratebypass=yes'
-                for stream in streams: # audio
-                    if stream['type'].startswith('audio/mp4'):
-                        dash_mp4_a_url = stream['url']
+                           for afmt in video_info['adaptive_fmts'][0].split(',')]
+
+            for stream in streams: # get over speed limiting
+                stream['url'] += '&ratebypass=yes'
+            for stream in streams: # audio
+                if stream['type'].startswith('audio/mp4'):
+                    dash_mp4_a_url = stream['url']
+                    if 's' in stream:
+                        sig = self.__class__.decipher(self.js, stream['s'])
+                        dash_mp4_a_url += '&signature={}'.format(sig)
+                    dash_mp4_a_size = stream['clen']
+                elif stream['type'].startswith('audio/webm'):
+                    dash_webm_a_url = stream['url']
+                    if 's' in stream:
+                        sig = self.__class__.decipher(self.js, stream['s'])
+                        dash_webm_a_url += '&signature={}'.format(sig)
+                    dash_webm_a_size = stream['clen']
+            for stream in streams: # video
+                if 'size' in stream:
+                    if stream['type'].startswith('video/mp4'):
+                        mimeType = 'video/mp4'
+                        dash_url = stream['url']
                         if 's' in stream:
                             sig = self.__class__.decipher(self.js, stream['s'])
-                            dash_mp4_a_url += '&signature={}'.format(sig)
-                        dash_mp4_a_size = stream['clen']
-                    elif stream['type'].startswith('audio/webm'):
-                        dash_webm_a_url = stream['url']
+                            dash_url += '&signature={}'.format(sig)
+                        dash_size = stream['clen']
+                        itag = stream['itag']
+                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                        dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
+                        self.dash_streams[itag] = {
+                            'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
+                            'itag': itag,
+                            'type': mimeType,
+                            'mime': mimeType,
+                            'container': 'mp4',
+                            'src': [dash_urls, dash_mp4_a_urls],
+                            'size': int(dash_size) + int(dash_mp4_a_size)
+                        }
+                    elif stream['type'].startswith('video/webm'):
+                        mimeType = 'video/webm'
+                        dash_url = stream['url']
                         if 's' in stream:
                             sig = self.__class__.decipher(self.js, stream['s'])
-                            dash_webm_a_url += '&signature={}'.format(sig)
-                        dash_webm_a_size = stream['clen']
-                for stream in streams: # video
-                    if 'size' in stream:
-                        if stream['type'].startswith('video/mp4'):
-                            mimeType = 'video/mp4'
-                            dash_url = stream['url']
-                            if 's' in stream:
-                                sig = self.__class__.decipher(self.js, stream['s'])
-                                dash_url += '&signature={}'.format(sig)
-                            dash_size = stream['clen']
-                            itag = stream['itag']
-                            dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
-                            dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
-                            self.dash_streams[itag] = {
-                                'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
-                                'itag': itag,
-                                'type': mimeType,
-                                'mime': mimeType,
-                                'container': 'mp4',
-                                'src': [dash_urls, dash_mp4_a_urls],
-                                'size': int(dash_size) + int(dash_mp4_a_size)
-                            }
-                        elif stream['type'].startswith('video/webm'):
-                            mimeType = 'video/webm'
-                            dash_url = stream['url']
-                            if 's' in stream:
-                                sig = self.__class__.decipher(self.js, stream['s'])
-                                dash_url += '&signature={}'.format(sig)
-                            dash_size = stream['clen']
-                            itag = stream['itag']
-                            audio_url = None
-                            audio_size = None
-                            try:
-                                audio_url = dash_webm_a_url
-                                audio_size = int(dash_webm_a_size)
-                            except UnboundLocalError as e:
-                                audio_url = dash_mp4_a_url
-                                audio_size = int(dash_mp4_a_size)
-                            dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
-                            audio_urls = self.__class__.chunk_by_range(audio_url, int(audio_size))
-                            self.dash_streams[itag] = {
-                                'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
-                                'itag': itag,
-                                'type': mimeType,
-                                'mime': mimeType,
-                                'container': 'webm',
-                                'src': [dash_urls, audio_urls],
-                                'size': int(dash_size) + int(audio_size)
-                            }
+                            dash_url += '&signature={}'.format(sig)
+                        dash_size = stream['clen']
+                        itag = stream['itag']
+                        audio_url = None
+                        audio_size = None
+                        try:
+                            audio_url = dash_webm_a_url
+                            audio_size = int(dash_webm_a_size)
+                        except UnboundLocalError as e:
+                            audio_url = dash_mp4_a_url
+                            audio_size = int(dash_mp4_a_size)
+                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
+                        audio_urls = self.__class__.chunk_by_range(audio_url, int(audio_size))
+                        self.dash_streams[itag] = {
+                            'quality': '%s (%s)' % (stream['size'], stream['quality_label']),
+                            'itag': itag,
+                            'type': mimeType,
+                            'mime': mimeType,
+                            'container': 'webm',
+                            'src': [dash_urls, audio_urls],
+                            'size': int(dash_size) + int(audio_size)
+                        }
 
     def extract(self, **kwargs):
         if not self.streams_sorted:

From 23fb59fe7be0273b2b4eb2ade96ff5d323cd5006 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 21 Apr 2019 00:54:09 +0200
Subject: [PATCH 0837/1225] [twitter] call universal_download for
 pbs.twimg.com/*

---
 src/you_get/extractors/twitter.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 1c02797310..ec1b06af15 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -3,6 +3,7 @@
 __all__ = ['twitter_download']
 
 from ..common import *
+from .universal import *
 from .vine import vine_download
 
 def extract_m3u(source):
@@ -15,6 +16,10 @@ def extract_m3u(source):
     return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    if re.match(r'https?://pbs\.twimg\.com', url):
+        universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
+        return
+
     if re.match(r'https?://mobile', url): # normalize mobile URL
         url = 'https://' + match1(url, r'//mobile\.(.+)')
 

From e6202412c328378091e4d22a4033906d62bf2d0a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 22 Apr 2019 02:04:39 +0200
Subject: [PATCH 0838/1225] [youtube] oopsie

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 0fcf2b6974..5000cc22da 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -419,7 +419,7 @@ def prepare(self, **kwargs):
                 streams = [dict([(i.split('=')[0],
                                   parse.unquote(i.split('=')[1]))
                                  for i in afmt.split('&')])
-                           for afmt in ytplayer_config['args']['adaptive_fmts'][0].split(',')]
+                           for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
             except:
                 streams = [dict([(i.split('=')[0],
                                   parse.unquote(i.split('=')[1]))

From 77aa95094bb5af9441f14d2ab7c2f6a199be7270 Mon Sep 17 00:00:00 2001
From: Wing Gao <wing.gao@live.com>
Date: Mon, 22 Apr 2019 14:24:07 +0800
Subject: [PATCH 0839/1225] add new video type for bilibili.com

---
 src/you_get/extractors/bilibili.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index cddd9d6f18..668f40f858 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -12,7 +12,8 @@ class Bilibili(VideoExtractor):
     stream_types = [
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
-        # 'id': 'hdflv2', 'quality': 112?
+        {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
         {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
         {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
@@ -111,6 +112,13 @@ def bilibili_space_video_api(mid, pn=1, ps=100):
     def bilibili_vc_api(video_id):
         return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
+    @staticmethod
+    def url_size(url, faker=False, headers={},err_value=0):
+        try:
+            return url_size(url,faker,headers)
+        except:
+            return err_value
+
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
@@ -244,7 +252,7 @@ def prepare(self, **kwargs):
                         desc = s['desc']
                         audio_quality = s['audio_quality']
                         baseurl = video['baseUrl']
-                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+                        size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
 
                         # find matching audio track
                         audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
@@ -253,7 +261,7 @@ def prepare(self, **kwargs):
                                 audio_baseurl = audio['baseUrl']
                                 break
                         if not audio_size_cache.get(audio_quality, False):
-                            audio_size_cache[audio_quality] = url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
                         size += audio_size_cache[audio_quality]
 
                         self.dash_streams[format_id] = {'container': container, 'quality': desc,

From 25b86beedec932d4a2f935cf534688925b5fdfb3 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@me.com>
Date: Wed, 17 Apr 2019 09:54:13 +0200
Subject: [PATCH 0840/1225] Travis CI: Add flake8 test F632 use ==/!= to
 compare str, bytes, and int literals

Also:
* Upgrade to Python 3.7 production instead of pre-release
* Upgrade to current Python nightly and 3.8-dev builds
* Remove __sudo__ tag which is now deprecated on Travis CI

[__Python < 3.5 are all end-of-life.__](https://devguide.python.org/devcycle/#end-of-life-branches)  At what point do we drop support for them?
---
 .travis.yml | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c11cbe34d2..735a6d917c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,15 +6,20 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "3.7-dev"
-  - "nightly"
   - "pypy3"
+matrix:
+  include:
+    - python: "3.7"
+      dist: xenial
+    - python: "3.8-dev"
+      dist: xenial    
+    - python: "nightly"
+      dist: xenial    
 before_install:
   - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then pip install flake8; fi
 before_script:
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics; fi
+  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics; fi
 script: make test
-sudo: false
 notifications:
   webhooks:
     urls:

From 2f5bb48601bab9b7a19a035da9a62ca6116cef10 Mon Sep 17 00:00:00 2001
From: cclauss <cclauss@me.com>
Date: Mon, 22 Apr 2019 09:38:05 +0200
Subject: [PATCH 0841/1225] Fix use ==/!= to compare str, bytes, and int
 literals

---
 src/you_get/extractors/baidu.py   | 2 +-
 src/you_get/extractors/douyutv.py | 4 ++--
 src/you_get/extractors/panda.py   | 6 +++---
 src/you_get/extractors/zhibo.py   | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index d3744f9dff..7914667e61 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -38,7 +38,7 @@ def baidu_get_song_title(data):
 
 def baidu_get_song_lyric(data):
     lrc = data['lrcLink']
-    return None if lrc is '' else "http://music.baidu.com%s" % lrc
+    return "http://music.baidu.com%s" % lrc if lrc else None
 
 
 def baidu_download_song(sid, output_dir='.', merge=True, info_only=False):
diff --git a/src/you_get/extractors/douyutv.py b/src/you_get/extractors/douyutv.py
index bdcea45827..82ea5c4dea 100644
--- a/src/you_get/extractors/douyutv.py
+++ b/src/you_get/extractors/douyutv.py
@@ -62,12 +62,12 @@ def douyutv_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     json_content = json.loads(content)
     data = json_content['data']
     server_status = json_content.get('error', 0)
-    if server_status is not 0:
+    if server_status != 0:
         raise ValueError("Server returned error:%s" % server_status)
 
     title = data.get('room_name')
     show_status = data.get('show_status')
-    if show_status is not "1":
+    if show_status != "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % server_status)
 
     real_url = data.get('rtmp_url') + '/' + data.get('rtmp_live')
diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
index ba6f2b4e1c..c9af4b38f0 100644
--- a/src/you_get/extractors/panda.py
+++ b/src/you_get/extractors/panda.py
@@ -15,7 +15,7 @@ def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     json_request_url ="http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}".format(roomid, int(time.time()))
     content = get_html(json_request_url)
     api_json = json.loads(content)
-    
+
     errno = api_json["errno"]
     errmsg = api_json["errmsg"]
     if errno:
@@ -25,7 +25,7 @@ def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     room_key = data["videoinfo"]["room_key"]
     plflag = data["videoinfo"]["plflag"].split("_")
     status = data["videoinfo"]["status"]
-    if status is not "2":
+    if status != "2":
         raise ValueError("The live stream is not online! (status:%s)" % status)
 
     data2 = json.loads(data["videoinfo"]["plflag_list"])
@@ -33,7 +33,7 @@ def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     sign = data2["auth"]["sign"]
     ts = data2["auth"]["time"]
     real_url = "http://pl{}.live.panda.tv/live_panda/{}.flv?sign={}&ts={}&rid={}".format(plflag[1], room_key, sign, ts, rid)
-    
+
     print_info(site_info, title, 'flv', float('inf'))
     if not info_only:
         download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
diff --git a/src/you_get/extractors/zhibo.py b/src/you_get/extractors/zhibo.py
index 1d2eadea79..a6143c308f 100644
--- a/src/you_get/extractors/zhibo.py
+++ b/src/you_get/extractors/zhibo.py
@@ -14,7 +14,7 @@ def zhibo_vedio_download(url, output_dir = '.', merge = True, info_only = False,
 
     video_html = r1(r'<script type="text/javascript">([\s\S]*)</script></head>', html)
 
-    # video_guessulike = r1(r"window.xgData =([s\S'\s\.]*)\'\;[\s\S]*window.vouchData", video_html) 
+    # video_guessulike = r1(r"window.xgData =([s\S'\s\.]*)\'\;[\s\S]*window.vouchData", video_html)
     video_url = r1(r"window.vurl = \'([s\S'\s\.]*)\'\;[\s\S]*window.imgurl", video_html)
     part_urls.append(video_url)
     ext = video_url.split('.')[-1]
@@ -34,7 +34,7 @@ def zhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwa
     html = get_html(url)
     title = r1(r'<title>([\s\S]*)</title>', html)
     is_live = r1(r"window.videoIsLive=\'([s\S'\s\.]*)\'\;[\s\S]*window.resDomain", html)
-    if is_live is not "1":
+    if is_live != "1":
         raise ValueError("The live stream is not online! (Errno:%s)" % is_live)
 
     match = re.search(r"""

From 2e5ced0700fe5cfff2f604bd9645158694b4e970 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Apr 2019 15:35:35 +0200
Subject: [PATCH 0842/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 9ecc662d78..2c388054d8 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -6,12 +6,10 @@
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url, faker=True)
-    title = r1(r'<title>(.*?)</title>', html)
+    title = r1(r'<title.*?>(.*?)</title>', html)
     video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
     title = '%s [%s]' % (title, video_id)
-    dataText = r1(r'var data = \[(.*)\] ', html) or r1(r'var data = (\{.*\})', html)
-    data = json.loads(dataText)
-    source = 'http:' + data['video']['play_addr']['url_list'][0]
+    source = r1(r'<video .*?src="([^"]+)"', html)
     mime, ext, size = url_info(source)
 
     print_info(site_info, title, mime, size)

From 82f2e1f3eb7e733a80dbad96259e585e176a0586 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Apr 2019 16:23:50 +0200
Subject: [PATCH 0843/1225] version 0.4.1295

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index d3c39cd09d..95bedb5da4 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1270'
+__version__ = '0.4.1295'

From d6f9fefe42cb44b8577a08c95c7af2ecf1238c50 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 26 Apr 2019 03:52:39 +0200
Subject: [PATCH 0844/1225] [miaopai] support stories

---
 src/you_get/extractors/miaopai.py | 33 +++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 01d043f268..0ddcadba0b 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -65,26 +65,47 @@ def miaopai_download_by_wbmp(wbmp_url, fid, info_only=False, **kwargs):
         download_urls([video_url], fs.legitimize(title), ext, headers=headers, **kwargs)
 
 
-def miaopai_download_direct(url, info_only, **kwargs):
+def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **kwargs):
+    data_url = 'https://m.weibo.cn/s/video/object?%s' % url.split('?')[1]
+    data_content = get_content(data_url, headers=fake_headers_mobile)
+    data = json.loads(data_content)
+    title = data['data']['object']['summary']
+    stream_url = data['data']['object']['stream']['url']
+
+    ext = 'mp4'
+    print_info(site_info, title, ext, url_info(stream_url, headers=fake_headers_mobile)[2])
+    if not info_only:
+        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
+
+
+def miaopai_download_direct(url, output_dir='.', merge=False, info_only=False, **kwargs):
     mobile_page = get_content(url, headers=fake_headers_mobile)
     try:
         title = re.search(r'([\'"])title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
     except:
         title = re.search(r'([\'"])status_title\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
     title = title.replace('\n', '_')
-    stream_url = re.search(r'([\'"])stream_url\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    try:
+        stream_url = re.search(r'([\'"])stream_url\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+    except:
+        page_url = re.search(r'([\'"])page_url\1:\s*([\'"])(.+?)\2,', mobile_page).group(3)
+        return miaopai_download_story(page_url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     ext = 'mp4'
     print_info(site_info, title, ext, url_info(stream_url, headers=fake_headers_mobile)[2])
     if not info_only:
         download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
 
 
-# ----------------------------------------------------------------------
-def miaopai_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
-    if match1(url, r'weibo\.com/tv/v/(\w+)'):
+def miaopai_download(url, output_dir='.', merge=False, info_only=False, **kwargs):
+    if re.match(r'^http[s]://.*\.weibo\.com/\d+/.+', url):
         return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
-    if re.match(r'^http[s]://.*\.weibo\.com/\d+/.+', url):
+    if re.match(r'^http[s]://.*\.weibo\.(com|cn)/s/video/.+', url):
+        return miaopai_download_story(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
+    # FIXME!
+    if re.match(r'^http[s]://.*\.weibo\.com/tv/v/(\w+)', url):
         return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
     fid = match1(url, r'\?fid=(\d{4}:\w+)')

From ad08f82a1a2d0b9847168f8c88941d1170f8a54d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 28 Apr 2019 19:02:45 +0200
Subject: [PATCH 0845/1225] [embed] get rid of empty src in iframe

---
 src/you_get/extractors/embed.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index 035a050b22..ee53974729 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -133,7 +133,7 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False, **kwa
             r = 1
         else:
             r += 1
-        iframes = matchall(content, [r'<iframe.+?src=(?:\"|\')(.+?)(?:\"|\')'])
+        iframes = matchall(content, [r'<iframe.+?src=(?:\"|\')(.*?)(?:\"|\')'])
         for iframe in iframes:
             if not iframe.startswith('http'):
                 src = urllib.parse.urljoin(url, iframe)

From b746ac01c9f39de94cac2d56f665285b0523b974 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Apr 2019 00:29:28 +0200
Subject: [PATCH 0846/1225] [universal] update

---
 src/you_get/extractors/universal.py | 46 +++++++++++++++++------------
 1 file changed, 27 insertions(+), 19 deletions(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 69ef5d90fb..8b9a24c954 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -33,27 +33,35 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         meta_videos = re.findall(r'<meta property="og:video:url" content="([^"]*)"', page)
         if meta_videos:
-            for meta_video in meta_videos:
-                meta_video_url = unescape_html(meta_video)
-                type_, ext, size = url_info(meta_video_url)
-                print_info(site_info, page_title, type_, size)
-                if not info_only:
-                    download_urls([meta_video_url], page_title,
-                                  ext, size,
-                                  output_dir=output_dir, merge=merge,
-                                  faker=True)
-            return
+            try:
+                for meta_video in meta_videos:
+                    meta_video_url = unescape_html(meta_video)
+                    type_, ext, size = url_info(meta_video_url)
+                    print_info(site_info, page_title, type_, size)
+                    if not info_only:
+                        download_urls([meta_video_url], page_title,
+                                      ext, size,
+                                      output_dir=output_dir, merge=merge,
+                                      faker=True)
+            except:
+                pass
+            else:
+                return
 
         hls_urls = re.findall(r'(https?://[^;"\'\\]+' + '\.m3u8?' +
                               r'[^;"\'\\]*)', page)
         if hls_urls:
-            for hls_url in hls_urls:
-                type_, ext, size = url_info(hls_url)
-                print_info(site_info, page_title, type_, size)
-                if not info_only:
-                    download_url_ffmpeg(url=hls_url, title=page_title,
-                                        ext='mp4', output_dir=output_dir)
-            return
+            try:
+                for hls_url in hls_urls:
+                    type_, ext, size = url_info(hls_url)
+                    print_info(site_info, page_title, type_, size)
+                    if not info_only:
+                        download_url_ffmpeg(url=hls_url, title=page_title,
+                                            ext='mp4', output_dir=output_dir)
+            except:
+                pass
+            else:
+                return
 
         # most common media file extensions on the Internet
         media_exts = ['\.flv', '\.mp3', '\.mp4', '\.webm',
@@ -67,12 +75,12 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^ ;&"\'\\]+' + i + r'[^ ;&"\'\\]*)', page)
+            urls += re.findall(r'(https?://[^ ;&"\'\\<>]+' + i + r'[^ ;&"\'\\<>]*)', page)
 
             p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page)
             urls += [parse.unquote(url) for url in p_urls]
 
-            q_urls = re.findall(r'(https?:\\\\/\\\\/[^ ;"\']+' + i + r'[^ ;"\']*)', page)
+            q_urls = re.findall(r'(https?:\\\\/\\\\/[^ ;"\'<>]+' + i + r'[^ ;"\'<>]*)', page)
             urls += [url.replace('\\\\/', '/') for url in q_urls]
 
         # a link href to an image is often an interesting one

From 8103f29b7815c36e92e13cb57031124d54a8a46d Mon Sep 17 00:00:00 2001
From: icemoon1987 <panwenhai1987@163.com>
Date: Mon, 29 Apr 2019 14:16:51 +0800
Subject: [PATCH 0847/1225] change --skip-ignore-size to
 --skip-existing-file-size-check

---
 src/you_get/common.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index bb9ec4a95d..aa68f07bf8 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -131,7 +131,7 @@
 dry_run = False
 json_output = False
 force = False
-skip_ignore_size = False
+skip_existing_file_size_check = False
 player = None
 extractor_proxy = None
 cookies = None
@@ -634,11 +634,11 @@ def url_save(
     while continue_renameing:
         continue_renameing = False
         if os.path.exists(filepath):
-            if not force and (file_size == os.path.getsize(filepath) or skip_ignore_size):
+            if not force and (file_size == os.path.getsize(filepath) or skip_existing_file_size_check):
                 if not is_part:
                     if bar:
                         bar.done()
-                    if skip_ignore_size:
+                    if skip_existing_file_size_check:
                         log.w(
                             'Skipping {} without checking size: file already exists'.format(
                                 tr(os.path.basename(filepath))
@@ -954,8 +954,8 @@ def download_urls(
     if total_size:
         if not force and os.path.exists(output_filepath) and not auto_rename\
                 and (os.path.getsize(output_filepath) >= total_size * 0.9\
-                or skip_ignore_size):
-            if skip_ignore_size:
+                or skip_existing_file_size_check):
+            if skip_existing_file_size_check:
                 log.w('Skipping %s without checking size: file already exists' % output_filepath)
             else:
                 log.w('Skipping %s: file already exists' % output_filepath)
@@ -1468,7 +1468,7 @@ def print_version():
         help='Force overwriting existing files'
     )
     download_grp.add_argument(
-        '--skip-ignore-size', action='store_true', default=False,
+        '--skip-existing-file-size-check', action='store_true', default=False,
         help='Skip existing file without checking file size'
     )
     download_grp.add_argument(
@@ -1557,7 +1557,7 @@ def print_version():
         logging.getLogger().setLevel(logging.DEBUG)
 
     global force
-    global skip_ignore_size
+    global skip_existing_file_size_check
     global dry_run
     global json_output
     global player
@@ -1571,8 +1571,8 @@ def print_version():
     info_only = args.info
     if args.force:
         force = True
-    if args.skip_ignore_size:
-        skip_ignore_size = True
+    if args.skip_existing_file_size_check:
+        skip_existing_file_size_check = True
     if args.auto_rename:
         auto_rename = True
     if args.url:

From fbf46b04537cb4033016aac810f823f2598a9260 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 9 May 2019 01:22:03 +0200
Subject: [PATCH 0848/1225] [youtube] fix #749, again (see also 1401f9a)

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 5000cc22da..0629035b96 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -109,7 +109,7 @@ def tr_js(code):
             else:
                 f2def = re.search(r'[^$\w]%s:function\((\w+)\)(\{[^\{\}]+\})' % f2e, js)
                 f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
-            f2 = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', f2)
+            f2 = re.sub(r'(as|if|in|is|or)', r'_\1', f2)
             f2 = re.sub(r'\$', '_dollar', f2)
             code = code + 'global %s\n' % f2 + tr_js(f2def)
 

From 8623c5aceb169263ed26ff9a64f8f44de4e8180c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 9 May 2019 01:35:12 +0200
Subject: [PATCH 0849/1225] remove python 3.2 & 3.3 for travis

---
 .travis.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 735a6d917c..8433fe75d3 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,8 +1,6 @@
 # https://travis-ci.org/soimort/you-get
 language: python
 python:
-  - "3.2"
-  - "3.3"
   - "3.4"
   - "3.5"
   - "3.6"
@@ -12,13 +10,13 @@ matrix:
     - python: "3.7"
       dist: xenial
     - python: "3.8-dev"
-      dist: xenial    
+      dist: xenial
     - python: "nightly"
-      dist: xenial    
+      dist: xenial
 before_install:
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then pip install flake8; fi
+  - pip install flake8
 before_script:
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.2'* && $TRAVIS_PYTHON_VERSION != '3.3'* ]]; then flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics; fi
+  - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics
 script: make test
 notifications:
   webhooks:

From bd699cbee3ec2d674758677f0323263204a138a2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 9 May 2019 01:39:55 +0200
Subject: [PATCH 0850/1225] [tests] remove test_toutiao since it breaks too
 often

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 20b8de5050..9584ac51df 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -32,9 +32,6 @@ def test_youtube(self):
             info_only=True
         )
 
-    def test_toutiao(self):
-        toutiao.download('https://www.365yg.com/i6640053613567675662/#mid=1611922564114440', info_only=True)
-
 
 if __name__ == '__main__':
     unittest.main()

From edbe9dd4c15f09f81818a53f89f3c4f5a8ce2374 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 9 May 2019 01:43:06 +0200
Subject: [PATCH 0851/1225] version 0.4.1302

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 95bedb5da4..654732ce47 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1295'
+__version__ = '0.4.1302'

From 3926c0ea02588a6c5a610e739d4a26dd8e6f7eff Mon Sep 17 00:00:00 2001
From: URenko <18209292+URenko@users.noreply.github.com>
Date: Thu, 9 May 2019 17:28:03 +0800
Subject: [PATCH 0852/1225] https support for AcFun

---
 src/you_get/extractors/acfun.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 424418adb8..88363ed5db 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -109,9 +109,9 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
             pass
 
 def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    assert re.match(r'http://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
+    assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
 
-    if re.match(r'http://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
+    if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
         html = get_content(url)
         title = r1(r'data-title="([^"]+)"', html)
         if match1(url, r'_(\d+)$'):  # current P
@@ -119,7 +119,7 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = r1('data-vid="(\d+)"', html)
         up = r1('data-name="([^"]+)"', html)
     # bangumi
-    elif re.match("http://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
+    elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url)
         title = match1(html, r'"title"\s*:\s*"([^"]+)"')
         if match1(url, r'_(\d+)$'):  # current P

From 6648e007e3b6d9d7b608d6b73c5e032cb00e0f59 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 19 May 2019 02:06:37 +0200
Subject: [PATCH 0853/1225] [common] set output_filename for multi-part
 downloads (longstanding bug since 20ebd90 and d8e2a93)

---
 src/you_get/common.py | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 87c31f8975..e469f7a174 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -878,13 +878,16 @@ def done(self):
         pass
 
 
-def get_output_filename(urls, title, ext, output_dir, merge):
+def get_output_filename(urls, title, ext, output_dir, merge, **kwargs):
     # lame hack for the --output-filename option
     global output_filename
     if output_filename:
+        result = output_filename
+        if kwargs.get('part', -1) >= 0:
+            result = '%s[%02d]' % (result, kwargs.get('part'))
         if ext:
-            return output_filename + '.' + ext
-        return output_filename
+            result = result + '.' + ext
+        return result
 
     merged_ext = ext
     if (len(urls) > 1) and merge:
@@ -964,16 +967,16 @@ def download_urls(
         bar.done()
     else:
         parts = []
-        print('Downloading %s.%s ...' % (tr(title), ext))
+        print('Downloading %s ...' % tr(output_filename))
         bar.update()
         for i, url in enumerate(urls):
-            filename = '%s[%02d].%s' % (title, i, ext)
-            filepath = os.path.join(output_dir, filename)
-            parts.append(filepath)
+            output_filename_i = get_output_filename(urls, title, ext, output_dir, merge, part=i)
+            output_filepath_i = os.path.join(output_dir, output_filename_i)
+            parts.append(output_filepath_i)
             # print 'Downloading %s [%s/%s]...' % (tr(filename), i + 1, len(urls))
             bar.update_piece(i + 1)
             url_save(
-                url, filepath, bar, refer=refer, is_part=True, faker=faker,
+                url, output_filepath_i, bar, refer=refer, is_part=True, faker=faker,
                 headers=headers, **kwargs
             )
         bar.done()

From fc10ea1d06de0108a949d4dd2d8da7c5d270f219 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 19 May 2019 16:38:16 +0200
Subject: [PATCH 0854/1225] [common] fix get_output_filename

---
 src/you_get/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 120dfe6096..c39624316c 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -894,7 +894,7 @@ def get_output_filename(urls, title, ext, output_dir, merge, **kwargs):
         if kwargs.get('part', -1) >= 0:
             result = '%s[%02d]' % (result, kwargs.get('part'))
         if ext:
-            result = result + '.' + ext
+            result = '%s.%s' % (result, ext)
         return result
 
     merged_ext = ext
@@ -912,7 +912,11 @@ def get_output_filename(urls, title, ext, output_dir, merge, **kwargs):
                 merged_ext = 'mkv'
             else:
                 merged_ext = 'ts'
-    return '%s.%s' % (title, merged_ext)
+    result = title
+    if kwargs.get('part', -1) >= 0:
+        result = '%s[%02d]' % (result, kwargs.get('part'))
+    result = '%s.%s' % (result, merged_ext)
+    return result
 
 def print_user_agent(faker=False):
     urllib_default_user_agent = 'Python-urllib/%d.%d' % sys.version_info[:2]

From 5ef3d2dc55c2aad32f133e5f918cb36c819200dd Mon Sep 17 00:00:00 2001
From: GuanFoxyier <1372851437@qq.com>
Date: Thu, 23 May 2019 10:49:07 +0800
Subject: [PATCH 0855/1225] fix letv

---
 src/you_get/extractors/le.py | 92 +++++++++++++++++++-----------------
 1 file changed, 49 insertions(+), 43 deletions(-)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index 940b9b350b..cd5b7a8a98 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -2,20 +2,23 @@
 
 __all__ = ['letv_download', 'letvcloud_download', 'letvcloud_download_by_vu']
 
-import json
+import base64
+import hashlib
 import random
-import xml.etree.ElementTree as ET
-import base64, hashlib, urllib, time, re
+import urllib
 
 from ..common import *
 
-#@DEPRECATED
+
+# @DEPRECATED
 def get_timestamp():
     tn = random.random()
     url = 'http://api.letv.com/time?tn={}'.format(tn)
     result = get_content(url)
     return json.loads(result)['stime']
-#@DEPRECATED
+
+
+# @DEPRECATED
 def get_key(t):
     for s in range(0, 8):
         e = 1 & t
@@ -24,42 +27,40 @@ def get_key(t):
         t += e
     return t ^ 185025305
 
+
 def calcTimeKey(t):
-    ror = lambda val, r_bits, : ((val & (2**32-1)) >> r_bits%32) |  (val << (32-(r_bits%32)) & (2**32-1))
+    ror = lambda val, r_bits,: ((val & (2 ** 32 - 1)) >> r_bits % 32) | (val << (32 - (r_bits % 32)) & (2 ** 32 - 1))
     magic = 185025305
     return ror(t, magic % 17) ^ magic
-    #return ror(ror(t,773625421%13)^773625421,773625421%17)
+    # return ror(ror(t,773625421%13)^773625421,773625421%17)
 
 
 def decode(data):
     version = data[0:5]
     if version.lower() == b'vc_01':
-        #get real m3u8
+        # get real m3u8
         loc2 = data[5:]
         length = len(loc2)
-        loc4 = [0]*(2*length)
+        loc4 = [0] * (2 * length)
         for i in range(length):
-            loc4[2*i] = loc2[i] >> 4
-            loc4[2*i+1]= loc2[i] & 15;
-        loc6 = loc4[len(loc4)-11:]+loc4[:len(loc4)-11]
-        loc7 = [0]*length
+            loc4[2 * i] = loc2[i] >> 4
+            loc4[2 * i + 1] = loc2[i] & 15;
+        loc6 = loc4[len(loc4) - 11:] + loc4[:len(loc4) - 11]
+        loc7 = [0] * length
         for i in range(length):
-            loc7[i] = (loc6[2 * i] << 4) +loc6[2*i+1]
+            loc7[i] = (loc6[2 * i] << 4) + loc6[2 * i + 1]
         return ''.join([chr(i) for i in loc7])
     else:
         # directly return
-        return data
-
-
+        return str(data)
 
 
-def video_info(vid,**kwargs):
-    url = 'http://player-pc.le.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.le.com&region=cn&source=1000&accesyx=1'.format(vid,calcTimeKey(int(time.time())))
+def video_info(vid, **kwargs):
+    url = 'http://player-pc.le.com/mms/out/video/playJson?id={}&platid=1&splatid=105&format=1&tkey={}&domain=www.le.com&region=cn&source=1000&accesyx=1'.format(vid, calcTimeKey(int(time.time())))
     r = get_content(url, decoded=False)
-    info=json.loads(str(r,"utf-8"))
+    info = json.loads(str(r, "utf-8"))
     info = info['msgs']
 
-
     stream_id = None
     support_stream_id = info["playurl"]["dispatch"].keys()
     if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id:
@@ -70,27 +71,28 @@ def video_info(vid,**kwargs):
         elif "720p" in support_stream_id:
             stream_id = '720p'
         else:
-            stream_id =sorted(support_stream_id,key= lambda i: int(i[1:]))[-1]
+            stream_id = sorted(support_stream_id, key=lambda i: int(i[1:]))[-1]
 
-    url =info["playurl"]["domain"][0]+info["playurl"]["dispatch"][stream_id][0]
+    url = info["playurl"]["domain"][0] + info["playurl"]["dispatch"][stream_id][0]
     uuid = hashlib.sha1(url.encode('utf8')).hexdigest() + '_0'
     ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1]
     url = url.replace('tss=0', 'tss=ios')
-    url+="&m3v=1&termid=1&format=1&hwtype=un&ostype=MacOS10.12.4&p1=1&p2=10&p3=-&expect=3&tn={}&vid={}&uuid={}&sign=letv".format(random.random(), vid, uuid)
+    url += "&m3v=1&termid=1&format=1&hwtype=un&ostype=MacOS10.12.4&p1=1&p2=10&p3=-&expect=3&tn={}&vid={}&uuid={}&sign=letv".format(random.random(), vid, uuid)
 
-    r2=get_content(url,decoded=False)
-    info2=json.loads(str(r2,"utf-8"))
+    r2 = get_content(url, decoded=False)
+    info2 = json.loads(str(r2, "utf-8"))
 
     # hold on ! more things to do
     # to decode m3u8 (encoded)
     suffix = '&r=' + str(int(time.time() * 1000)) + '&appid=500'
-    m3u8 = get_content(info2["location"]+suffix,decoded=False)
+    m3u8 = get_content(info2["location"] + suffix, decoded=False)
     m3u8_list = decode(m3u8)
-    urls = re.findall(r'^[^#][^\r]*',m3u8_list,re.MULTILINE)
-    return ext,urls
+    urls = re.findall(r'(http.*?)#', m3u8_list, re.MULTILINE)
+    return ext, urls
+
 
-def letv_download_by_vid(vid,title, output_dir='.', merge=True, info_only=False,**kwargs):
-    ext , urls = video_info(vid,**kwargs)
+def letv_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
+    ext, urls = video_info(vid, **kwargs)
     size = 0
     for i in urls:
         _, _, tmp = url_info(i)
@@ -100,27 +102,29 @@ def letv_download_by_vid(vid,title, output_dir='.', merge=True, info_only=False,
     if not info_only:
         download_urls(urls, title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def letvcloud_download_by_vu(vu, uu, title=None, output_dir='.', merge=True, info_only=False):
-    #ran = float('0.' + str(random.randint(0, 9999999999999999))) # For ver 2.1
-    #str2Hash = 'cfflashformatjsonran{ran}uu{uu}ver2.2vu{vu}bie^#@(%27eib58'.format(vu = vu, uu = uu, ran = ran)  #Magic!/ In ver 2.1
-    argumet_dict ={'cf' : 'flash', 'format': 'json', 'ran': str(int(time.time())), 'uu': str(uu),'ver': '2.2', 'vu': str(vu), }
-    sign_key = '2f9d6924b33a165a6d8b5d3d42f4f987'  #ALL YOUR BASE ARE BELONG TO US
+    # ran = float('0.' + str(random.randint(0, 9999999999999999))) # For ver 2.1
+    # str2Hash = 'cfflashformatjsonran{ran}uu{uu}ver2.2vu{vu}bie^#@(%27eib58'.format(vu = vu, uu = uu, ran = ran)  #Magic!/ In ver 2.1
+    argumet_dict = {'cf': 'flash', 'format': 'json', 'ran': str(int(time.time())), 'uu': str(uu), 'ver': '2.2', 'vu': str(vu), }
+    sign_key = '2f9d6924b33a165a6d8b5d3d42f4f987'  # ALL YOUR BASE ARE BELONG TO US
     str2Hash = ''.join([i + argumet_dict[i] for i in sorted(argumet_dict)]) + sign_key
     sign = hashlib.md5(str2Hash.encode('utf-8')).hexdigest()
-    request_info = urllib.request.Request('http://api.letvcloud.com/gpc.php?' + '&'.join([i + '=' + argumet_dict[i] for i in argumet_dict]) + '&sign={sign}'.format(sign = sign))
+    request_info = urllib.request.Request('http://api.letvcloud.com/gpc.php?' + '&'.join([i + '=' + argumet_dict[i] for i in argumet_dict]) + '&sign={sign}'.format(sign=sign))
     response = urllib.request.urlopen(request_info)
     data = response.read()
     info = json.loads(data.decode('utf-8'))
     type_available = []
     for video_type in info['data']['video_info']['media']:
         type_available.append({'video_url': info['data']['video_info']['media'][video_type]['play_url']['main_url'], 'video_quality': int(info['data']['video_info']['media'][video_type]['play_url']['vtype'])})
-    urls = [base64.b64decode(sorted(type_available, key = lambda x:x['video_quality'])[-1]['video_url']).decode("utf-8")]
+    urls = [base64.b64decode(sorted(type_available, key=lambda x: x['video_quality'])[-1]['video_url']).decode("utf-8")]
     size = urls_size(urls)
     ext = 'mp4'
     print_info(site_info, title, ext, size)
     if not info_only:
         download_urls(urls, title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def letvcloud_download(url, output_dir='.', merge=True, info_only=False):
     qs = parse.urlparse(url).query
     vu = match1(qs, r'vu=([\w]+)')
@@ -128,7 +132,8 @@ def letvcloud_download(url, output_dir='.', merge=True, info_only=False):
     title = "LETV-%s" % vu
     letvcloud_download_by_vu(vu, uu, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
+
+def letv_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = url_locations([url])[0]
     if re.match(r'http://yuntv.letv.com/', url):
         letvcloud_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
@@ -136,14 +141,15 @@ def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
         html = get_content(url)
         vid = match1(url, r'video/(\d+)\.html')
         title = match1(html, r'<h2 class="title">([^<]+)</h2>')
-        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
+        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
     else:
         html = get_content(url)
         vid = match1(url, r'http://www.letv.com/ptv/vplay/(\d+).html') or \
-            match1(url, r'http://www.le.com/ptv/vplay/(\d+).html') or \
-            match1(html, r'vid="(\d+)"')
-        title = match1(html,r'name="irTitle" content="(.*?)"')
-        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
+              match1(url, r'http://www.le.com/ptv/vplay/(\d+).html') or \
+              match1(html, r'vid="(\d+)"')
+        title = match1(html, r'name="irTitle" content="(.*?)"')
+        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
 
 site_info = "Le.com"
 download = letv_download

From 8eb703edbeb5629bc513a0c7f9201465a617145f Mon Sep 17 00:00:00 2001
From: GuanFoxyier <1372851437@qq.com>
Date: Thu, 23 May 2019 11:12:49 +0800
Subject: [PATCH 0856/1225] fix le

---
 src/you_get/extractors/le.py | 86 +++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 40 deletions(-)

diff --git a/src/you_get/extractors/le.py b/src/you_get/extractors/le.py
index 940b9b350b..af49c2dcec 100644
--- a/src/you_get/extractors/le.py
+++ b/src/you_get/extractors/le.py
@@ -9,13 +9,16 @@
 
 from ..common import *
 
-#@DEPRECATED
+
+# @DEPRECATED
 def get_timestamp():
     tn = random.random()
     url = 'http://api.letv.com/time?tn={}'.format(tn)
     result = get_content(url)
     return json.loads(result)['stime']
-#@DEPRECATED
+
+
+# @DEPRECATED
 def get_key(t):
     for s in range(0, 8):
         e = 1 & t
@@ -24,42 +27,40 @@ def get_key(t):
         t += e
     return t ^ 185025305
 
+
 def calcTimeKey(t):
-    ror = lambda val, r_bits, : ((val & (2**32-1)) >> r_bits%32) |  (val << (32-(r_bits%32)) & (2**32-1))
+    ror = lambda val, r_bits,: ((val & (2 ** 32 - 1)) >> r_bits % 32) | (val << (32 - (r_bits % 32)) & (2 ** 32 - 1))
     magic = 185025305
     return ror(t, magic % 17) ^ magic
-    #return ror(ror(t,773625421%13)^773625421,773625421%17)
+    # return ror(ror(t,773625421%13)^773625421,773625421%17)
 
 
 def decode(data):
     version = data[0:5]
     if version.lower() == b'vc_01':
-        #get real m3u8
+        # get real m3u8
         loc2 = data[5:]
         length = len(loc2)
-        loc4 = [0]*(2*length)
+        loc4 = [0] * (2 * length)
         for i in range(length):
-            loc4[2*i] = loc2[i] >> 4
-            loc4[2*i+1]= loc2[i] & 15;
-        loc6 = loc4[len(loc4)-11:]+loc4[:len(loc4)-11]
-        loc7 = [0]*length
+            loc4[2 * i] = loc2[i] >> 4
+            loc4[2 * i + 1] = loc2[i] & 15;
+        loc6 = loc4[len(loc4) - 11:] + loc4[:len(loc4) - 11]
+        loc7 = [0] * length
         for i in range(length):
-            loc7[i] = (loc6[2 * i] << 4) +loc6[2*i+1]
+            loc7[i] = (loc6[2 * i] << 4) + loc6[2 * i + 1]
         return ''.join([chr(i) for i in loc7])
     else:
         # directly return
-        return data
-
-
+        return str(data)
 
 
-def video_info(vid,**kwargs):
-    url = 'http://player-pc.le.com/mms/out/video/playJson?id={}&platid=1&splatid=101&format=1&tkey={}&domain=www.le.com&region=cn&source=1000&accesyx=1'.format(vid,calcTimeKey(int(time.time())))
+def video_info(vid, **kwargs):
+    url = 'http://player-pc.le.com/mms/out/video/playJson?id={}&platid=1&splatid=105&format=1&tkey={}&domain=www.le.com&region=cn&source=1000&accesyx=1'.format(vid, calcTimeKey(int(time.time())))
     r = get_content(url, decoded=False)
-    info=json.loads(str(r,"utf-8"))
+    info = json.loads(str(r, "utf-8"))
     info = info['msgs']
 
-
     stream_id = None
     support_stream_id = info["playurl"]["dispatch"].keys()
     if "stream_id" in kwargs and kwargs["stream_id"].lower() in support_stream_id:
@@ -70,27 +71,28 @@ def video_info(vid,**kwargs):
         elif "720p" in support_stream_id:
             stream_id = '720p'
         else:
-            stream_id =sorted(support_stream_id,key= lambda i: int(i[1:]))[-1]
+            stream_id = sorted(support_stream_id, key=lambda i: int(i[1:]))[-1]
 
-    url =info["playurl"]["domain"][0]+info["playurl"]["dispatch"][stream_id][0]
+    url = info["playurl"]["domain"][0] + info["playurl"]["dispatch"][stream_id][0]
     uuid = hashlib.sha1(url.encode('utf8')).hexdigest() + '_0'
     ext = info["playurl"]["dispatch"][stream_id][1].split('.')[-1]
     url = url.replace('tss=0', 'tss=ios')
-    url+="&m3v=1&termid=1&format=1&hwtype=un&ostype=MacOS10.12.4&p1=1&p2=10&p3=-&expect=3&tn={}&vid={}&uuid={}&sign=letv".format(random.random(), vid, uuid)
+    url += "&m3v=1&termid=1&format=1&hwtype=un&ostype=MacOS10.12.4&p1=1&p2=10&p3=-&expect=3&tn={}&vid={}&uuid={}&sign=letv".format(random.random(), vid, uuid)
 
-    r2=get_content(url,decoded=False)
-    info2=json.loads(str(r2,"utf-8"))
+    r2 = get_content(url, decoded=False)
+    info2 = json.loads(str(r2, "utf-8"))
 
     # hold on ! more things to do
     # to decode m3u8 (encoded)
     suffix = '&r=' + str(int(time.time() * 1000)) + '&appid=500'
-    m3u8 = get_content(info2["location"]+suffix,decoded=False)
+    m3u8 = get_content(info2["location"] + suffix, decoded=False)
     m3u8_list = decode(m3u8)
-    urls = re.findall(r'^[^#][^\r]*',m3u8_list,re.MULTILINE)
-    return ext,urls
+    urls = re.findall(r'(http.*?)#', m3u8_list, re.MULTILINE)
+    return ext, urls
+
 
-def letv_download_by_vid(vid,title, output_dir='.', merge=True, info_only=False,**kwargs):
-    ext , urls = video_info(vid,**kwargs)
+def letv_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
+    ext, urls = video_info(vid, **kwargs)
     size = 0
     for i in urls:
         _, _, tmp = url_info(i)
@@ -100,27 +102,29 @@ def letv_download_by_vid(vid,title, output_dir='.', merge=True, info_only=False,
     if not info_only:
         download_urls(urls, title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def letvcloud_download_by_vu(vu, uu, title=None, output_dir='.', merge=True, info_only=False):
-    #ran = float('0.' + str(random.randint(0, 9999999999999999))) # For ver 2.1
-    #str2Hash = 'cfflashformatjsonran{ran}uu{uu}ver2.2vu{vu}bie^#@(%27eib58'.format(vu = vu, uu = uu, ran = ran)  #Magic!/ In ver 2.1
-    argumet_dict ={'cf' : 'flash', 'format': 'json', 'ran': str(int(time.time())), 'uu': str(uu),'ver': '2.2', 'vu': str(vu), }
-    sign_key = '2f9d6924b33a165a6d8b5d3d42f4f987'  #ALL YOUR BASE ARE BELONG TO US
+    # ran = float('0.' + str(random.randint(0, 9999999999999999))) # For ver 2.1
+    # str2Hash = 'cfflashformatjsonran{ran}uu{uu}ver2.2vu{vu}bie^#@(%27eib58'.format(vu = vu, uu = uu, ran = ran)  #Magic!/ In ver 2.1
+    argumet_dict = {'cf': 'flash', 'format': 'json', 'ran': str(int(time.time())), 'uu': str(uu), 'ver': '2.2', 'vu': str(vu), }
+    sign_key = '2f9d6924b33a165a6d8b5d3d42f4f987'  # ALL YOUR BASE ARE BELONG TO US
     str2Hash = ''.join([i + argumet_dict[i] for i in sorted(argumet_dict)]) + sign_key
     sign = hashlib.md5(str2Hash.encode('utf-8')).hexdigest()
-    request_info = urllib.request.Request('http://api.letvcloud.com/gpc.php?' + '&'.join([i + '=' + argumet_dict[i] for i in argumet_dict]) + '&sign={sign}'.format(sign = sign))
+    request_info = urllib.request.Request('http://api.letvcloud.com/gpc.php?' + '&'.join([i + '=' + argumet_dict[i] for i in argumet_dict]) + '&sign={sign}'.format(sign=sign))
     response = urllib.request.urlopen(request_info)
     data = response.read()
     info = json.loads(data.decode('utf-8'))
     type_available = []
     for video_type in info['data']['video_info']['media']:
         type_available.append({'video_url': info['data']['video_info']['media'][video_type]['play_url']['main_url'], 'video_quality': int(info['data']['video_info']['media'][video_type]['play_url']['vtype'])})
-    urls = [base64.b64decode(sorted(type_available, key = lambda x:x['video_quality'])[-1]['video_url']).decode("utf-8")]
+    urls = [base64.b64decode(sorted(type_available, key=lambda x: x['video_quality'])[-1]['video_url']).decode("utf-8")]
     size = urls_size(urls)
     ext = 'mp4'
     print_info(site_info, title, ext, size)
     if not info_only:
         download_urls(urls, title, ext, size, output_dir=output_dir, merge=merge)
 
+
 def letvcloud_download(url, output_dir='.', merge=True, info_only=False):
     qs = parse.urlparse(url).query
     vu = match1(qs, r'vu=([\w]+)')
@@ -128,7 +132,8 @@ def letvcloud_download(url, output_dir='.', merge=True, info_only=False):
     title = "LETV-%s" % vu
     letvcloud_download_by_vu(vu, uu, title=title, output_dir=output_dir, merge=merge, info_only=info_only)
 
-def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
+
+def letv_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = url_locations([url])[0]
     if re.match(r'http://yuntv.letv.com/', url):
         letvcloud_download(url, output_dir=output_dir, merge=merge, info_only=info_only)
@@ -136,14 +141,15 @@ def letv_download(url, output_dir='.', merge=True, info_only=False ,**kwargs):
         html = get_content(url)
         vid = match1(url, r'video/(\d+)\.html')
         title = match1(html, r'<h2 class="title">([^<]+)</h2>')
-        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
+        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
     else:
         html = get_content(url)
         vid = match1(url, r'http://www.letv.com/ptv/vplay/(\d+).html') or \
-            match1(url, r'http://www.le.com/ptv/vplay/(\d+).html') or \
-            match1(html, r'vid="(\d+)"')
-        title = match1(html,r'name="irTitle" content="(.*?)"')
-        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only,**kwargs)
+              match1(url, r'http://www.le.com/ptv/vplay/(\d+).html') or \
+              match1(html, r'vid="(\d+)"')
+        title = match1(html, r'name="irTitle" content="(.*?)"')
+        letv_download_by_vid(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
+
 
 site_info = "Le.com"
 download = letv_download

From 5e5d2540d5bcff8f1e88427b7147ce3d255e5818 Mon Sep 17 00:00:00 2001
From: GuanFoxyier <1372851437@qq.com>
Date: Fri, 24 May 2019 14:33:17 +0800
Subject: [PATCH 0857/1225] fix sohu

---
 src/you_get/extractors/sohu.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 607bf44c6b..63971bc36f 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -15,11 +15,13 @@
         new api
 '''
 
+
 def real_url(fileName, key, ch):
     url = "https://data.vod.itc.cn/ip?new=" + fileName + "&num=1&key=" + key + "&ch=" + ch + "&pt=1&pg=2&prod=h5n"
     return json.loads(get_html(url))['servers'][0]['url']
 
-def sohu_download(url, output_dir = '.', merge = True, info_only = False, extractor_proxy=None, **kwargs):
+
+def sohu_download(url, output_dir='.', merge=True, info_only=False, extractor_proxy=None, **kwargs):
     if re.match(r'http://share.vrs.sohu.com', url):
         vid = r1('id=(\d+)', url)
     else:
@@ -31,12 +33,12 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         if extractor_proxy:
             set_proxy(tuple(extractor_proxy.split(":")))
         info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
-        for qtyp in ["oriVid","superVid","highVid" ,"norVid","relativeId"]:
+        for qtyp in ["oriVid", "superVid", "highVid", "norVid", "relativeId"]:
             if 'data' in info:
                 hqvid = info['data'][qtyp]
             else:
                 hqvid = info[qtyp]
-            if hqvid != 0 and hqvid != vid :
+            if hqvid != 0 and hqvid != vid:
                 info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % hqvid))
                 if not 'allot' in info:
                     continue
@@ -63,14 +65,15 @@ def sohu_download(url, output_dir = '.', merge = True, info_only = False, extrac
         urls = []
         data = info['data']
         title = data['tvName']
-        size = sum(map(int,data['clipsBytes']))
+        size = sum(map(int, data['clipsBytes']))
         assert len(data['clipsURL']) == len(data['clipsBytes']) == len(data['su'])
         for fileName, key in zip(data['su'], data['ck']):
             urls.append(real_url(fileName, key, data['ch']))
 
     print_info(site_info, title, 'mp4', size)
     if not info_only:
-        download_urls(urls, title, 'mp4', size, output_dir, refer = url, merge = merge)
+        download_urls(urls, title, 'mp4', size, output_dir, refer=url, merge=merge)
+
 
 site_info = "Sohu.com"
 download = sohu_download

From 8c41765d1364fe77900cdbcb8e9ebf99757b625f Mon Sep 17 00:00:00 2001
From: GuanFoxyier <1372851437@qq.com>
Date: Fri, 24 May 2019 14:35:49 +0800
Subject: [PATCH 0858/1225] fix sohu

---
 src/you_get/extractors/sohu.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 63971bc36f..a1afc1265c 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -29,10 +29,10 @@ def sohu_download(url, output_dir='.', merge=True, info_only=False, extractor_pr
         vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
     assert vid
 
-    if re.match(r'http[s]://tv.sohu.com/', url):
-        if extractor_proxy:
-            set_proxy(tuple(extractor_proxy.split(":")))
-        info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
+    if extractor_proxy:
+        set_proxy(tuple(extractor_proxy.split(":")))
+    info = json.loads(get_decoded_html('http://hot.vrs.sohu.com/vrs_flash.action?vid=%s' % vid))
+    if info and info.get("data", ""):
         for qtyp in ["oriVid", "superVid", "highVid", "norVid", "relativeId"]:
             if 'data' in info:
                 hqvid = info['data'][qtyp]

From 1758cd0ae99b15047cef9f4ca4ba0bce18866ccd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Jun 2019 15:56:46 +0200
Subject: [PATCH 0859/1225] [youtube] signature -> sig (fix #2711)

---
 src/you_get/extractors/youtube.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 0629035b96..eea31503cd 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -216,6 +216,7 @@ def prepare(self, **kwargs):
                     self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                     # Workaround: get_video_info returns bad s. Why?
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
+                    #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
                 except:
                     stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
                     if re.search('([^"]*/base\.js)"', video_page):
@@ -306,7 +307,8 @@ def prepare(self, **kwargs):
                 'url': metadata['url'][0],
                 'sig': metadata['sig'][0] if 'sig' in metadata else None,
                 's': metadata['s'][0] if 's' in metadata else None,
-                'quality': metadata['quality'][0],
+                'quality': metadata['quality'][0] if 'quality' in metadata else None,
+                #'quality': metadata['quality_label'][0] if 'quality_label' in metadata else None,
                 'type': metadata['type'][0],
                 'mime': metadata['type'][0].split(';')[0],
                 'container': mime_to_container(metadata['type'][0].split(';')[0]),
@@ -433,13 +435,13 @@ def prepare(self, **kwargs):
                     dash_mp4_a_url = stream['url']
                     if 's' in stream:
                         sig = self.__class__.decipher(self.js, stream['s'])
-                        dash_mp4_a_url += '&signature={}'.format(sig)
+                        dash_mp4_a_url += '&sig={}'.format(sig)
                     dash_mp4_a_size = stream['clen']
                 elif stream['type'].startswith('audio/webm'):
                     dash_webm_a_url = stream['url']
                     if 's' in stream:
                         sig = self.__class__.decipher(self.js, stream['s'])
-                        dash_webm_a_url += '&signature={}'.format(sig)
+                        dash_webm_a_url += '&sig={}'.format(sig)
                     dash_webm_a_size = stream['clen']
             for stream in streams: # video
                 if 'size' in stream:
@@ -448,7 +450,7 @@ def prepare(self, **kwargs):
                         dash_url = stream['url']
                         if 's' in stream:
                             sig = self.__class__.decipher(self.js, stream['s'])
-                            dash_url += '&signature={}'.format(sig)
+                            dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
                         dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
@@ -467,7 +469,7 @@ def prepare(self, **kwargs):
                         dash_url = stream['url']
                         if 's' in stream:
                             sig = self.__class__.decipher(self.js, stream['s'])
-                            dash_url += '&signature={}'.format(sig)
+                            dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
                         audio_url = None
@@ -510,13 +512,13 @@ def extract(self, **kwargs):
             src = self.streams[stream_id]['url']
             if self.streams[stream_id]['sig'] is not None:
                 sig = self.streams[stream_id]['sig']
-                src += '&signature={}'.format(sig)
+                src += '&sig={}'.format(sig)
             elif self.streams[stream_id]['s'] is not None:
                 if not hasattr(self, 'js'):
                     self.js = get_content(self.html5player)
                 s = self.streams[stream_id]['s']
                 sig = self.__class__.decipher(self.js, s)
-                src += '&signature={}'.format(sig)
+                src += '&sig={}'.format(sig)
 
             self.streams[stream_id]['src'] = [src]
             self.streams[stream_id]['size'] = urls_size(self.streams[stream_id]['src'])

From 2496decf24a8c8228137fdaf2391bcc6f74a9207 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Jun 2019 16:02:36 +0200
Subject: [PATCH 0860/1225] version 0.4.1314

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 654732ce47..a31efa487d 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1302'
+__version__ = '0.4.1314'

From fe7279ab47a25d5bb8a6c4aea6e55397c74a6c31 Mon Sep 17 00:00:00 2001
From: GuanFoxyier <1372851437@qq.com>
Date: Mon, 17 Jun 2019 10:16:17 +0800
Subject: [PATCH 0861/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=85=B7=E7=8B=97?=
 =?UTF-8?q?=E9=9F=B3=E4=B9=90=E6=97=A0=E6=B3=95=E4=B8=8B=E8=BD=BD=E9=97=AE?=
 =?UTF-8?q?=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/kugou.py | 77 ++++++++++++++++++---------------
 1 file changed, 41 insertions(+), 36 deletions(-)

diff --git a/src/you_get/extractors/kugou.py b/src/you_get/extractors/kugou.py
index a98daac6d5..192bd809d7 100644
--- a/src/you_get/extractors/kugou.py
+++ b/src/you_get/extractors/kugou.py
@@ -8,66 +8,72 @@
 import re
 import hashlib
 
+
 def kugou_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
-    if url.lower().find("5sing")!=-1:
-        #for 5sing.kugou.com
-        html=get_html(url)
-        ticket=r1(r'"ticket":\s*"(.*)"',html)
-        j=loads(str(b64decode(ticket),encoding="utf-8"))
-        url=j['file']
-        title=j['songName']
+    if url.lower().find("5sing") != -1:
+        # for 5sing.kugou.com
+        html = get_html(url)
+        ticket = r1(r'"ticket":\s*"(.*)"', html)
+        j = loads(str(b64decode(ticket), encoding="utf-8"))
+        url = j['file']
+        title = j['songName']
         songtype, ext, size = url_info(url)
         print_info(site_info, title, songtype, size)
         if not info_only:
             download_urls([url], title, ext, size, output_dir, merge=merge)
-    elif url.lower().find("hash")!=-1:
-        return kugou_download_by_hash(url,output_dir,merge,info_only)
+    elif url.lower().find("hash") != -1:
+        return kugou_download_by_hash(url, output_dir, merge, info_only)
     else:
-        #for the www.kugou.com/
+        # for the www.kugou.com/
         return kugou_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
         # raise NotImplementedError(url)       
 
 
-def kugou_download_by_hash(url,output_dir = '.', merge = True, info_only = False):
-    #sample
-    #url_sample:http://www.kugou.com/song/#hash=93F7D2FC6E95424739448218B591AEAF&album_id=9019462
-    hash_val = match1(url,'hash=(\w+)')
-    album_id = match1(url,'album_id=(\d+)')
-    html = get_html("http://www.kugou.com/yy/index.php?r=play/getdata&hash={}&album_id={}".format(hash_val,album_id))
-    j =loads(html)
+def kugou_download_by_hash(url, output_dir='.', merge=True, info_only=False):
+    # sample
+    # url_sample:http://www.kugou.com/song/#hash=93F7D2FC6E95424739448218B591AEAF&album_id=9019462
+    hash_val = match1(url, 'hash=(\w+)')
+    album_id = match1(url, 'album_id=(\d+)')
+    if not album_id:
+        album_id = 123
+    html = get_html("http://www.kugou.com/yy/index.php?r=play/getdata&hash={}&album_id={}&mid=123".format(hash_val, album_id))
+    j = loads(html)
     url = j['data']['play_url']
     title = j['data']['audio_name']
     # some songs cann't play because of copyright protection
-    if(url == ''):
+    if (url == ''):
         return
     songtype, ext, size = url_info(url)
     print_info(site_info, title, songtype, size)
     if not info_only:
         download_urls([url], title, ext, size, output_dir, merge=merge)
 
-def kugou_download_playlist(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    urls=[]
-    
-    #download music leaderboard
-    #sample: http://www.kugou.com/yy/html/rank.html
-    if url.lower().find('rank') !=-1:
-        html=get_html(url)
+
+def kugou_download_playlist(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    urls = []
+
+    # download music leaderboard
+    # sample: http://www.kugou.com/yy/html/rank.html
+    if url.lower().find('rank') != -1:
+        html = get_html(url)
         pattern = re.compile('<a href="(http://.*?)" data-active=')
         res = pattern.findall(html)
         for song in res:
             res = get_html(song)
             pattern_url = re.compile('"hash":"(\w+)".*"album_id":(\d)+')
-            hash_val,album_id= res = pattern_url.findall(res)[0]
-            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(hash_val,album_id))
-    
+            hash_val, album_id = res = pattern_url.findall(res)[0]
+            if not album_id:
+                album_id = 123
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s' % (hash_val, album_id))
+
     # download album
     # album sample:   http://www.kugou.com/yy/album/single/1645030.html
-    elif url.lower().find('album')!=-1:
+    elif url.lower().find('album') != -1:
         html = get_html(url)
         pattern = re.compile('var data=(\[.*?\]);')
         res = pattern.findall(html)[0]
         for v in json.loads(res):
-            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v['hash'],v['album_id']))
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v['hash'], v['album_id']))
 
     # download the playlist        
     # playlist sample:http://www.kugou.com/yy/special/single/487279.html
@@ -75,16 +81,15 @@ def kugou_download_playlist(url, output_dir = '.', merge = True, info_only = Fal
         html = get_html(url)
         pattern = re.compile('data="(\w+)\|(\d+)"')
         for v in pattern.findall(html):
-            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v[0],v[1]))
-            print('http://www.kugou.com/song/#hash=%s&album_id=%s'%(v[0],v[1]))
+            urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v[0], v[1]))
+            print('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v[0], v[1]))
 
-    #download the list by hash
+    # download the list by hash
     for url in urls:
-        kugou_download_by_hash(url,output_dir,merge,info_only)
+        kugou_download_by_hash(url, output_dir, merge, info_only)
 
-                
 
 site_info = "kugou.com"
 download = kugou_download
 # download_playlist = playlist_not_supported("kugou")
-download_playlist=kugou_download_playlist
+download_playlist = kugou_download_playlist

From c617e68efa54059c2318545568754c6bfa644b8a Mon Sep 17 00:00:00 2001
From: Will Han <xingheng.hax@qq.com>
Date: Tue, 2 Jul 2019 10:46:34 +0800
Subject: [PATCH 0862/1225] Fix the parser issue for acfun videos.

---
 src/you_get/extractors/acfun.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 88363ed5db..3dacedf52a 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -113,11 +113,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
     if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
         html = get_content(url)
-        title = r1(r'data-title="([^"]+)"', html)
-        if match1(url, r'_(\d+)$'):  # current P
-            title = title + " " + r1(r'active">([^<]*)', html)
-        vid = r1('data-vid="(\d+)"', html)
-        up = r1('data-name="([^"]+)"', html)
+        json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
+        json_data = json.loads(json_text)
+        vid = json_data.get('currentVideoInfo').get('id')
+        up = json_data.get('user').get('name')
+        title = json_data.get('title')
+        video_list = json_data.get('videoList')
+        if len(video_list) > 1:
+            title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
     # bangumi
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url)

From 80671b86ae2e25d9d3a6b3ce46f3e49b10293368 Mon Sep 17 00:00:00 2001
From: Tianran YAO <48848200+yaotianran@users.noreply.github.com>
Date: Tue, 16 Jul 2019 09:45:54 +0800
Subject: [PATCH 0863/1225] changed youku ccode to 0519

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 75a49c707e..d7107eca24 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -78,7 +78,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0590'
+        self.ccode = '0519'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 179bbeaa5e46048cc963d1c4923fcf2893e76908 Mon Sep 17 00:00:00 2001
From: laiqing <laiqing@staff.sina.com.cn>
Date: Tue, 16 Jul 2019 12:48:55 +0800
Subject: [PATCH 0864/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A5=BF=E7=93=9C?=
 =?UTF-8?q?=E8=A7=86=E9=A2=91=E8=8E=B7=E5=8F=96=E4=B8=8D=E5=88=B0video=20i?=
 =?UTF-8?q?d=E5=92=8Ctitle=E7=9A=84=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/ixigua.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 3cf07b094e..164161ccf7 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -81,8 +81,8 @@ def tmp():
 def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
     html = get_html(url, faker=True)
-    video_id = match1(html, r"videoId\s*:\s*'([^']+)'")
-    title = match1(html, r"title: '(\S+)',")
+    video_id = match1(html, r"\"vid\":\"([^\"]+)")
+    title = match1(html, r"\"title\":\"(\S+?)\",")
     if not video_id:
         log.e("video_id not found, url:{}".format(url))
         return

From 151938e1aa28c6d88ffca1e0edeff287239d7e26 Mon Sep 17 00:00:00 2001
From: qiaoruntao <925783095@qq.com>
Date: Tue, 16 Jul 2019 13:36:08 +0800
Subject: [PATCH 0865/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BF=AB=E6=89=8B?=
 =?UTF-8?q?=E7=9B=B4=E6=92=AD=E6=97=A0=E6=B3=95=E4=B8=8B=E8=BD=BD=E7=9A=84?=
 =?UTF-8?q?=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/kuaishou.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/kuaishou.py b/src/you_get/extractors/kuaishou.py
index a21f8ffa64..917920d196 100644
--- a/src/you_get/extractors/kuaishou.py
+++ b/src/you_get/extractors/kuaishou.py
@@ -16,11 +16,14 @@ def kuaishou_download_by_url(url, info_only=False, **kwargs):
     # size = video_list[-1]['size']
     # result wrong size
     try:
-        og_video_url = re.search(r"<meta\s+property=\"og:video:url\"\s+content=\"(.+?)\"/>", page).group(1)
-        video_url = og_video_url
-        title = url.split('/')[-1]
+        search_result=re.search(r"\"playUrls\":\[(\{\"quality\"\:\"\w+\",\"url\":\".*?\"\})+\]", page)
+        all_video_info_str = search_result.group(1)
+        all_video_infos=re.findall(r"\{\"quality\"\:\"(\w+)\",\"url\":\"(.*?)\"\}", all_video_info_str)
+        # get the one of the best quality
+        video_url = all_video_infos[0][1].encode("utf-8").decode('unicode-escape')
+        title = re.search(r"<meta charset=UTF-8><title>(.*?)</title>", page).group(1)
         size = url_size(video_url)
-        video_format = video_url.split('.')[-1]
+        video_format = "flv"#video_url.split('.')[-1]
         print_info(site_info, title, video_format, size)
         if not info_only:
             download_urls([video_url], title, video_format, size, **kwargs)

From d3d397178fb2b24a7d0aabf11a995e293c968ebf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Jul 2019 23:08:23 +0800
Subject: [PATCH 0866/1225] [common] update UA

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c39624316c..24681b2279 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -144,7 +144,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:60.0) Gecko/20100101 Firefox/60.0',  # noqa
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0',  # noqa
 }
 
 if sys.stdout.isatty():

From e36675ce72f120ab7d053eeaab9c1268e11ed18e Mon Sep 17 00:00:00 2001
From: Ensteinjun <luoming_jun@126.com>
Date: Tue, 30 Jul 2019 15:09:55 +0800
Subject: [PATCH 0867/1225] Fix bug: unable get video title

---
 src/you_get/extractors/youtube.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index eea31503cd..976d270bce 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -207,8 +207,7 @@ def prepare(self, **kwargs):
             raise
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
-                self.title = parse.unquote_plus(video_info['title'][0])
-
+                self.title = parse.unquote_plus(json.loads(video_info["player_response"][0])["videoDetails"]["title"])
                 # Parse video page (for DASH)
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 try:
@@ -229,7 +228,7 @@ def prepare(self, **kwargs):
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
 
-                self.title = ytplayer_config['args']['title']
+                self.title = json.loads(ytplayer_config["args"]["player_response"])["videoDetails"]["title"]
                 self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                 stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
 

From 648487b9a256ffa1d9ba91758e0c8afe8409fb9b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 2 Aug 2019 13:30:10 +0200
Subject: [PATCH 0868/1225] version 0.4.1328

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index a31efa487d..48bf3b5f98 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1314'
+__version__ = '0.4.1328'

From 0fe204ad06cd96726ad9f770936ef961ad8bb12c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Aug 2019 10:31:49 +0200
Subject: [PATCH 0869/1225] [youtube] warn when target URL is from a playlist
 and --playlist is not used

---
 src/you_get/extractors/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 976d270bce..4483f8eb85 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -195,6 +195,9 @@ def prepare(self, **kwargs):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
+        if re.search('\Wlist=', self.url) and not kwargs.get('playlist'):
+            log.w('This video is from a playlist. (use --playlist to download all videos in the playlist.)')
+
         # Get video info
         # 'eurl' is a magic parameter that can bypass age restriction
         # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'

From 2a1db6e79c5465808b908d4b3b6c874c0d7a7cc9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Aug 2019 13:53:57 +0200
Subject: [PATCH 0870/1225] [panda] purge

---
 src/you_get/common.py              |  1 -
 src/you_get/extractors/__init__.py |  1 -
 src/you_get/extractors/panda.py    | 43 ------------------------------
 3 files changed, 45 deletions(-)
 delete mode 100644 src/you_get/extractors/panda.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 24681b2279..b2bca0a56a 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -86,7 +86,6 @@
     'naver'            : 'naver',
     '7gogo'            : 'nanagogo',
     'nicovideo'        : 'nicovideo',
-    'panda'            : 'panda',
     'pinterest'        : 'pinterest',
     'pixnet'           : 'pixnet',
     'pptv'             : 'pptv',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 0c4cccc72d..2961f01522 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -51,7 +51,6 @@
 from .naver import *
 from .netease import *
 from .nicovideo import *
-from .panda import *
 from .pinterest import *
 from .pixnet import *
 from .pptv import *
diff --git a/src/you_get/extractors/panda.py b/src/you_get/extractors/panda.py
deleted file mode 100644
index c9af4b38f0..0000000000
--- a/src/you_get/extractors/panda.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['panda_download']
-
-from ..common import *
-from ..util.log import *
-import json
-import time
-
-def panda_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    roomid = re.search('/(\d+)', url)
-    if roomid is None:
-        log.wtf('Cannot found room id for this url')
-    roomid = roomid.group(1)
-    json_request_url ="http://www.panda.tv/api_room_v2?roomid={}&__plat=pc_web&_={}".format(roomid, int(time.time()))
-    content = get_html(json_request_url)
-    api_json = json.loads(content)
-
-    errno = api_json["errno"]
-    errmsg = api_json["errmsg"]
-    if errno:
-        raise ValueError("Errno : {}, Errmsg : {}".format(errno, errmsg))
-    data = api_json["data"]
-    title = data["roominfo"]["name"]
-    room_key = data["videoinfo"]["room_key"]
-    plflag = data["videoinfo"]["plflag"].split("_")
-    status = data["videoinfo"]["status"]
-    if status != "2":
-        raise ValueError("The live stream is not online! (status:%s)" % status)
-
-    data2 = json.loads(data["videoinfo"]["plflag_list"])
-    rid = data2["auth"]["rid"]
-    sign = data2["auth"]["sign"]
-    ts = data2["auth"]["time"]
-    real_url = "http://pl{}.live.panda.tv/live_panda/{}.flv?sign={}&ts={}&rid={}".format(plflag[1], room_key, sign, ts, rid)
-
-    print_info(site_info, title, 'flv', float('inf'))
-    if not info_only:
-        download_urls([real_url], title, 'flv', None, output_dir, merge = merge)
-
-site_info = "panda.tv"
-download = panda_download
-download_playlist = playlist_not_supported('panda')

From 4ca4a51507ce2450f22f7ad2bf000fcfd48b06ac Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 3 Aug 2019 15:53:37 +0200
Subject: [PATCH 0871/1225] [twitter] disable faker to prevent 302 infinite
 redirect

---
 src/you_get/extractors/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index ec1b06af15..4b239e679b 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -34,7 +34,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    html = get_html(url, faker=True)
+    html = get_html(url, faker=False) # disable faker to prevent 302 infinite redirect
     screen_name = r1(r'twitter\.com/([^/]+)', url) or r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
     item_id = r1(r'twitter\.com/[^/]+/status/(\d+)', url) or r1(r'data-item-id="([^"]*)"', html) or \

From 4587fd913dff1c222cab9835e1f91ca1bc540535 Mon Sep 17 00:00:00 2001
From: Semen Zhydenko <simeon.zhidenko@gmail.com>
Date: Mon, 5 Aug 2019 09:52:38 +0200
Subject: [PATCH 0872/1225] avalable -> available

---
 src/you_get/extractors/mgtv.py | 2 +-
 src/you_get/extractors/qie.py  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 730dfeba44..657167a65f 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -68,7 +68,7 @@ def prepare(self, **kwargs):
         self.title = content['data']['info']['title']
         domain = content['data']['stream_domain'][0]
         
-        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        #stream_available = [i['name'] for i in content['data']['stream']]
         stream_available = {}
         for i in content['data']['stream']:
             stream_available[i['name']] = i['url']
diff --git a/src/you_get/extractors/qie.py b/src/you_get/extractors/qie.py
index 38f703ed27..08d462bf0e 100644
--- a/src/you_get/extractors/qie.py
+++ b/src/you_get/extractors/qie.py
@@ -58,7 +58,7 @@ def prepare(self, **kwargs):
         content = loads(content)
         self.title = content['data']['room_name']
         rtmp_url =  content['data']['rtmp_url']
-        #stream_avalable = [i['name'] for i in content['data']['stream']]
+        #stream_available = [i['name'] for i in content['data']['stream']]
         stream_available = {}
         stream_available['normal'] = rtmp_url + '/' + content['data']['rtmp_live']
         if len(content['data']['rtmp_multi_bitrate']) > 0:

From 2b80c6de9dad29fdd4a07633e741950fe51738f7 Mon Sep 17 00:00:00 2001
From: Semen Zhydenko <simeon.zhidenko@gmail.com>
Date: Mon, 5 Aug 2019 09:53:01 +0200
Subject: [PATCH 0873/1225] basicly -> basically

---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 02ecb012c2..63679b834d 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -220,7 +220,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
 def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=True):
     """str, str->True
     WARNING: NOT THE SAME PARMS AS OTHER FUNCTIONS!!!!!!
-    You can basicly download anything with this function
+    You can basically download anything with this function
     but better leave it alone with
     """
     output = title + '.' + ext

From 013e75c886160ded6ce62aea8db13860fd6e206c Mon Sep 17 00:00:00 2001
From: Semen Zhydenko <simeon.zhidenko@gmail.com>
Date: Mon, 5 Aug 2019 09:53:23 +0200
Subject: [PATCH 0874/1225] infomation -> information

---
 src/you_get/extractors/flickr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index 4efa78ef45..2535dd1cb7 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -74,7 +74,7 @@ def get_api_key(page):
     # this happens only when the url points to a gallery page
     # that contains no inline api_key(and never makes xhr api calls)
     # in fact this might be a better approch for getting a temporary api key
-    # since there's no place for a user to add custom infomation that may
+    # since there's no place for a user to add custom information that may
     # misguide the regex in the homepage
     if not match:
         return match1(get_html('https://flickr.com'), pattern_inline_api_key)

From be931a5416ad170ae719fb9d7e6b0f7b1b8ceef4 Mon Sep 17 00:00:00 2001
From: Semen Zhydenko <simeon.zhidenko@gmail.com>
Date: Mon, 5 Aug 2019 09:53:54 +0200
Subject: [PATCH 0875/1225] Unkown -> Unknown

---
 src/you_get/extractors/icourses.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/icourses.py b/src/you_get/extractors/icourses.py
index ec70f64cba..606e21e6e5 100644
--- a/src/you_get/extractors/icourses.py
+++ b/src/you_get/extractors/icourses.py
@@ -110,7 +110,7 @@ def icourses_playlist_download(url, output_dir='.', **kwargs):
         video_list = re.findall(resid_courseid_patt, page)
 
     if not video_list:
-        raise Exception('Unkown url pattern')
+        raise Exception('Unknown url pattern')
 
     for video in video_list:
         video_url = change_for_video_ip.format(video[0], video[1])

From aa151acaa3ee2ecb4a603aa82f87019a42dc2508 Mon Sep 17 00:00:00 2001
From: lxfly2000 <gaoboyuan8@qq.com>
Date: Sat, 10 Aug 2019 19:31:29 +0800
Subject: [PATCH 0876/1225] Fix AcFun Bangumi download.

---
 README.md                       |  2 +-
 src/you_get/extractors/acfun.py | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index e1551c9a54..360b5d0b25 100644
--- a/README.md
+++ b/README.md
@@ -403,7 +403,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **niconico<br/>ニコニコ動画** | <http://www.nicovideo.jp/> |✓| | |
 | **163<br/>网易视频<br/>网易云音乐** | <http://v.163.com/><br/><http://music.163.com/> |✓| |✓|
 | 56网     | <http://www.56.com/>           |✓| | |
-| **AcFun** | <http://www.acfun.tv/>        |✓| | |
+| **AcFun** | <http://www.acfun.cn/>        |✓| | |
 | **Baidu<br/>百度贴吧** | <http://tieba.baidu.com/> |✓|✓| |
 | 爆米花网 | <http://www.baomihua.com/>     |✓| | |
 | **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 3dacedf52a..61f6cae8f6 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -124,10 +124,11 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # bangumi
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url)
-        title = match1(html, r'"title"\s*:\s*"([^"]+)"')
-        if match1(url, r'_(\d+)$'):  # current P
-            title = title + " " + r1(r'active">([^<]*)', html)
-        vid = match1(html, r'videoId="(\d+)"')
+        tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
+        json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
+        json_data = json.loads(json_text)
+        title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
+        vid = str(json_data['videoId'])
         up = "acfun"
     else:
         raise NotImplemented
@@ -148,6 +149,6 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
                           **kwargs)
 
 
-site_info = "AcFun.tv"
+site_info = "AcFun.cn"
 download = acfun_download
 download_playlist = playlist_not_supported('acfun')

From 09982a36ce1b03f525cdd77bef775c5833b2fcee Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Mon, 12 Aug 2019 05:54:07 +0800
Subject: [PATCH 0877/1225] works again

---
 src/you_get/extractors/qq_egame.py | 59 ++++++++++++++++--------------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/src/you_get/extractors/qq_egame.py b/src/you_get/extractors/qq_egame.py
index 4ec36ef20d..c8dca6e0ba 100644
--- a/src/you_get/extractors/qq_egame.py
+++ b/src/you_get/extractors/qq_egame.py
@@ -1,7 +1,7 @@
 import re
 import json
 
-from ..common import get_content
+from ..common import *
 from ..extractors import VideoExtractor
 from ..util import log
 from ..util.strings import unescape_html
@@ -9,33 +9,36 @@
 __all__ = ['qq_egame_download']
 
 
-class QQEgame(VideoExtractor):
-    stream_types = [
-        {'id': 'original', 'video_profile': '0', 'container': 'flv'},
-        {'id': '900', 'video_profile': '900kb/s', 'container': 'flv'},
-        {'id': '550', 'video_profile': '550kb/s', 'container': 'flv'}
-    ]
-    name = 'QQEgame'
+def qq_egame_download(url,
+                      output_dir='.',
+                      merge=True,
+                      info_only=False,
+                      **kwargs):
+    uid = re.search('\d\d\d+', url)
+    an_url = "https://m.egame.qq.com/live?anchorid={}&".format(uid.group(0))
+    page = get_content(an_url)
+    server_data = re.search(r'window\.serverData\s*=\s*({.+?});', page)
+    if server_data is None:
+        log.wtf('Can not find window.server_data')
+    json_data = json.loads(server_data.group(1))
+    if json_data['anchorInfo']['data']['isLive'] == 0:
+        log.wtf('Offline...')
+    live_info = json_data['liveInfo']['data']
+    title = '{}_{}'.format(live_info['profileInfo']['nickName'],
+                           live_info['videoInfo']['title'])
+    real_url = live_info['videoInfo']['streamInfos'][0]['playUrl']
 
-    def prepare(self, **kwargs):
-        page = get_content(self.url)
-        server_data = re.search(r'serverData\s*=\s*({.+?});', page)
-        if server_data is None:
-            log.wtf('cannot find server_data')
-        json_data = json.loads(server_data.group(1))
-        live_info = json_data['liveInfo']['data']
-        self.title = '{}_{}'.format(live_info['profileInfo']['nickName'], live_info['videoInfo']['title'])
-        for exsited_stream in live_info['videoInfo']['streamInfos']:
-            for s in self.__class__.stream_types:
-                if re.search(r'(\d+)', s['video_profile']).group(1) == exsited_stream['bitrate']:
-                    current_stream_id = s['id']
-                    stream_info = dict(src=[unescape_html(exsited_stream['playUrl'])])
-                    stream_info['video_profile'] = exsited_stream['desc']
-                    stream_info['container'] = s['container']
-                    stream_info['size'] = float('inf')
-                    self.streams[current_stream_id] = stream_info
+    print_info(site_info, title, 'flv', float('inf'))
+    if not info_only:
+        download_url_ffmpeg(
+            real_url,
+            title,
+            'flv',
+            params={},
+            output_dir=output_dir,
+            merge=merge)
 
 
-def qq_egame_download(url, **kwargs):
-    QQEgame().download_by_url(url, **kwargs)
-    # url dispatching has been done in qq.py
+site_info = "egame.qq.com"
+download = qq_egame_download
+download_playlist = playlist_not_supported('qq_egame')

From 804eb35ac8b3b805a792451ce90410e0b80556f5 Mon Sep 17 00:00:00 2001
From: Justsoos <justso@gmail.com>
Date: Mon, 12 Aug 2019 05:55:32 +0800
Subject: [PATCH 0878/1225] for egame

---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index e39bf2e30d..fc4f3088cc 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -108,7 +108,7 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
 def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     """"""
 
-    if re.match(r'https?://egame.qq.com/live\?anchorid=(\d+)', url):
+    if re.match(r'https?://(m\.)?egame.qq.com/', url):
         from . import qq_egame
         qq_egame.qq_egame_download(url, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
         return

From 154b22930b356630a34c860b045cdbcd16cd85fc Mon Sep 17 00:00:00 2001
From: axzxc1236 <axzxc1236@gmail.com>
Date: Sun, 18 Aug 2019 08:10:15 +0800
Subject: [PATCH 0879/1225] Pick best video quality for ixigua

---
 src/you_get/extractors/ixigua.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 164161ccf7..11db08ac2b 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -103,10 +103,11 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         log.e("Get video info from {} error: The server returns JSON value"
               " without data.video_list.video_1 or data.video_list.video_1 is empty".format(video_info_url))
         return
-    size = int(video_info["data"]["video_list"]["video_1"]["size"])
+    bestQualityVideo = list(video_info["data"]["video_list"].keys())[-1] #There is not only video_1, there might be video_2
+    size = int(video_info["data"]["video_list"][bestQualityVideo]["size"])
     print_info(site_info=site_info, title=title, type="mp4", size=size)  # 该网站只有mp4类型文件
     if not info_only:
-        video_url = base64.b64decode(video_info["data"]["video_list"]["video_1"]["main_url"].encode("utf-8"))
+        video_url = base64.b64decode(video_info["data"]["video_list"][bestQualityVideo]["main_url"].encode("utf-8"))
         download_urls([video_url.decode("utf-8")], title, "mp4", size, output_dir, merge=merge, headers=headers, **kwargs)
 
 

From 65293201f3bbd599dc5626db630d68e8971a10fa Mon Sep 17 00:00:00 2001
From: axzxc1236 <axzxc1236@gmail.com>
Date: Sun, 18 Aug 2019 08:45:24 +0800
Subject: [PATCH 0880/1225] Fix wrong video title for ixigua

---
 src/you_get/extractors/ixigua.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 11db08ac2b..6ac252ef30 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -82,7 +82,7 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
     html = get_html(url, faker=True)
     video_id = match1(html, r"\"vid\":\"([^\"]+)")
-    title = match1(html, r"\"title\":\"(\S+?)\",")
+    title = match1(html, r"\"player__videoTitle\"><h1>(.*)<\/h1><\/div>")
     if not video_id:
         log.e("video_id not found, url:{}".format(url))
         return

From 0ec2a6ee56a0b522050b5059983fd11575dac64a Mon Sep 17 00:00:00 2001
From: chenxing <l>
Date: Wed, 28 Aug 2019 10:20:55 +0800
Subject: [PATCH 0881/1225] modify encoding with open cookies file

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index b2bca0a56a..9409462da3 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1317,7 +1317,7 @@ def load_cookies(cookiefile):
         cookies = cookiejar.MozillaCookieJar()
         now = time.time()
         ignore_discard, ignore_expires = False, False
-        with open(cookiefile, 'r') as f:
+        with open(cookiefile, 'r', encoding='utf-8') as f:
             for line in f:
                 # last field may be absent, so keep any trailing tab
                 if line.endswith("\n"): line = line[:-1]

From 06b9d97382a7b718d644ddaaa40a981e30b48dac Mon Sep 17 00:00:00 2001
From: FSpark <stardust@fspark.me>
Date: Mon, 9 Sep 2019 17:39:41 +0200
Subject: [PATCH 0882/1225] [bilibili] Fix bilibili download 403

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 668f40f858..b207a2b36a 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -45,7 +45,7 @@ def height_to_quality(height):
     def bilibili_headers(referer=None, cookie=None):
         # a reasonable UA
         ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
-        headers = {'User-Agent': ua}
+        headers = {'Accept': '*/*', 'User-Agent': ua}
         if referer is not None:
             headers.update({'Referer': referer})
         if cookie is not None:

From fea965d47ccbd703f34f44e44f234945339193f5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 9 Sep 2019 23:07:18 +0200
Subject: [PATCH 0883/1225] [acfun] fix (partly) #2734 (bangumi support still
 TBD)

---
 src/you_get/extractors/acfun.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 61f6cae8f6..df85add582 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -49,7 +49,7 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
     """
 
     #first call the main parasing API
-    info = json.loads(get_content('http://www.acfun.cn/video/getVideo.aspx?id=' + vid))
+    info = json.loads(get_content('http://www.acfun.cn/video/getVideo.aspx?id=' + vid, headers=fake_headers))
 
     sourceType = info['sourceType']
 
@@ -112,7 +112,7 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
 
     if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
-        html = get_content(url)
+        html = get_content(url, headers=fake_headers)
         json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
         json_data = json.loads(json_text)
         vid = json_data.get('currentVideoInfo').get('id')
@@ -121,7 +121,10 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         video_list = json_data.get('videoList')
         if len(video_list) > 1:
             title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
-    # bangumi
+
+        m3u8_url = json_data.get('currentVideoInfo')['playInfos'][0]['playUrls'][0]
+
+    # FIXME: bangumi
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url)
         tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
@@ -130,10 +133,11 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
         vid = str(json_data['videoId'])
         up = "acfun"
+
     else:
         raise NotImplemented
 
-    assert title and vid
+    assert title and m3u8_url
     title = unescape_html(title)
     title = escape_file_path(title)
     p_title = r1('active">([^<]+)', html)
@@ -141,12 +145,9 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if p_title:
         title = '%s - %s' % (title, p_title)
 
-
-    acfun_download_by_vid(vid, title,
-                          output_dir=output_dir,
-                          merge=merge,
-                          info_only=info_only,
-                          **kwargs)
+    print_info(site_info, title, 'm3u8', float('inf'))
+    if not info_only:
+        download_url_ffmpeg(m3u8_url, title, 'mp4', output_dir=output_dir, merge=merge)
 
 
 site_info = "AcFun.cn"

From d645eb7d70f76c9974da51d4517c77c1cc2c575a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 9 Sep 2019 23:14:04 +0200
Subject: [PATCH 0884/1225] version 0.4.1347

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 48bf3b5f98..c22ea56468 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1328'
+__version__ = '0.4.1347'

From 42924dcad5d05c831c550380a850379992cae796 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 11 Sep 2019 18:26:29 +0200
Subject: [PATCH 0885/1225] [bilibili] squanch this

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index b207a2b36a..2710b3329d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -45,7 +45,7 @@ def height_to_quality(height):
     def bilibili_headers(referer=None, cookie=None):
         # a reasonable UA
         ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
-        headers = {'Accept': '*/*', 'User-Agent': ua}
+        headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
         if referer is not None:
             headers.update({'Referer': referer})
         if cookie is not None:

From f58a1c08bf91c0917bf3cc321bc021523ff37ce2 Mon Sep 17 00:00:00 2001
From: "jiang.ma" <mailofmj@gmail.com>
Date: Tue, 17 Sep 2019 09:59:15 +0800
Subject: [PATCH 0886/1225] [ixigua] Fix error of matching title

---
 src/you_get/extractors/ixigua.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 6ac252ef30..20e4561618 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -82,7 +82,7 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
     html = get_html(url, faker=True)
     video_id = match1(html, r"\"vid\":\"([^\"]+)")
-    title = match1(html, r"\"player__videoTitle\"><h1>(.*)<\/h1><\/div>")
+    title = match1(html, r"\"player__videoTitle\">.*?<h1.*?>(.*)<\/h1><\/div>")
     if not video_id:
         log.e("video_id not found, url:{}".format(url))
         return

From 6efb4b0d436b82d762c3aa7a72bf5620872d46c3 Mon Sep 17 00:00:00 2001
From: lxfly2000 <gaoboyuan8@qq.com>
Date: Tue, 17 Sep 2019 18:19:36 +0800
Subject: [PATCH 0887/1225] Change acfun.py

---
 src/you_get/extractors/acfun.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index df85add582..52fcb4f91e 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -124,9 +124,8 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         m3u8_url = json_data.get('currentVideoInfo')['playInfos'][0]['playUrls'][0]
 
-    # FIXME: bangumi
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
-        html = get_content(url)
+        html = get_content(url, headers=fake_headers)
         tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
         json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
         json_data = json.loads(json_text)
@@ -134,6 +133,19 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         vid = str(json_data['videoId'])
         up = "acfun"
 
+        play_info = get_content("https://www.acfun.cn/rest/pc-direct/play/playInfo/m3u8Auto?videoId=" + vid, headers=fake_headers)
+        play_url = json.loads(play_info)['playInfo']['streams'][0]['playUrls'][0]
+        m3u8_all_qualities_file = get_content(play_url)
+        m3u8_all_qualities_lines = m3u8_all_qualities_file.split('#EXT-X-STREAM-INF:')[1:]
+        highest_quality_line = m3u8_all_qualities_lines[0]
+        for line in m3u8_all_qualities_lines:
+            bandwith = int(match1(line, r'BANDWIDTH=(\d+)'))
+            if bandwith > int(match1(highest_quality_line, r'BANDWIDTH=(\d+)')):
+                highest_quality_line = line
+        #TODO: 应由用户指定清晰度
+        m3u8_url = match1(highest_quality_line, r'\n([^#\n]+)$')
+        m3u8_url = play_url[:play_url.rfind("/")+1] + m3u8_url
+
     else:
         raise NotImplemented
 

From b5fe9103590f41d5fe17102424c9036f87d608dd Mon Sep 17 00:00:00 2001
From: Ensteinjun <luoming_jun@126.com>
Date: Wed, 18 Sep 2019 16:24:55 +0800
Subject: [PATCH 0888/1225] get all streams we can download

---
 src/you_get/json_output.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index d447ea2f56..0fcbc1c67f 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -11,6 +11,7 @@ def output(video_extractor, pretty_print=True):
     out['title'] = ve.title
     out['site'] = ve.name
     out['streams'] = ve.streams
+    out['streams'].update(ve.dash_streams)
     try:
         if ve.audiolang:
             out['audiolang'] = ve.audiolang

From 8fbd5f398299ad134526fc082d52f759ea6adb7f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 24 Sep 2019 11:24:34 +0200
Subject: [PATCH 0889/1225] [json_output] use dash_streams only if it exists

---
 src/you_get/json_output.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 0fcbc1c67f..5971bd939b 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -11,7 +11,11 @@ def output(video_extractor, pretty_print=True):
     out['title'] = ve.title
     out['site'] = ve.name
     out['streams'] = ve.streams
-    out['streams'].update(ve.dash_streams)
+    try:
+        if ve.dash_streams:
+            out['streams'].update(ve.dash_streams)
+    except AttributeError:
+        pass
     try:
         if ve.audiolang:
             out['audiolang'] = ve.audiolang
@@ -60,4 +64,3 @@ def download_urls(urls=None, title=None, ext=None, total_size=None, refer=None):
     ve.streams = {}
     ve.streams['__default__'] = stream
     output(ve)
-

From a38355711d6fcda64082ef28e42f99bd9ae3dfd0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 24 Sep 2019 13:44:06 +0200
Subject: [PATCH 0890/1225] version 0.4.1355

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index c22ea56468..1d87177c19 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1347'
+__version__ = '0.4.1355'

From 0d63b1d2870232c5da5b92f1de350ea643bba318 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Wed, 2 Oct 2019 18:05:38 +0800
Subject: [PATCH 0891/1225] Update bilibili.py

---
 src/you_get/extractors/bilibili.py | 1299 ++++++++++++++++------------
 1 file changed, 726 insertions(+), 573 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 2710b3329d..25cc0df3a7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -1,573 +1,726 @@
-#!/usr/bin/env python
-
-from ..common import *
-from ..extractor import VideoExtractor
-
-import hashlib
-
-class Bilibili(VideoExtractor):
-    name = "Bilibili"
-
-    # Bilibili media encoding options, in descending quality order.
-    stream_types = [
-        {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
-        {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
-        {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
-        {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
-        {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
-        {'id': 'hdmp4', 'quality': 48, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P (MP4)'},
-        {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
-        {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
-         'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
-        # 'quality': 15?
-        {'id': 'mp4', 'quality': 0},
-    ]
-
-    @staticmethod
-    def height_to_quality(height):
-        if height <= 360:
-            return 16
-        elif height <= 480:
-            return 32
-        elif height <= 720:
-            return 64
-        else:
-            return 80
-
-    @staticmethod
-    def bilibili_headers(referer=None, cookie=None):
-        # a reasonable UA
-        ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
-        headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
-        if referer is not None:
-            headers.update({'Referer': referer})
-        if cookie is not None:
-            headers.update({'Cookie': cookie})
-        return headers
-
-    @staticmethod
-    def bilibili_api(avid, cid, qn=0):
-        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
-
-    @staticmethod
-    def bilibili_audio_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/url?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_info_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_menu_info_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/menu/info?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_menu_song_api(sid, ps=100):
-        return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
-
-    @staticmethod
-    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
-        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
-
-    @staticmethod
-    def bilibili_interface_api(cid, qn=0):
-        entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
-        appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
-        params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
-        chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
-        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
-
-    @staticmethod
-    def bilibili_live_api(cid):
-        return 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid=%s&quality=0&platform=web' % cid
-
-    @staticmethod
-    def bilibili_live_room_info_api(room_id):
-        return 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id=%s' % room_id
-
-    @staticmethod
-    def bilibili_live_room_init_api(room_id):
-        return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
-
-    @staticmethod
-    def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
-        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
-
-    @staticmethod
-    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
-        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
-
-    @staticmethod
-    def bilibili_space_video_api(mid, pn=1, ps=100):
-        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
-
-    @staticmethod
-    def bilibili_vc_api(video_id):
-        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
-
-    @staticmethod
-    def url_size(url, faker=False, headers={},err_value=0):
-        try:
-            return url_size(url,faker,headers)
-        except:
-            return err_value
-
-    def prepare(self, **kwargs):
-        self.stream_qualities = {s['quality']: s for s in self.stream_types}
-
-        try:
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-        except:
-            html_content = ''  # live always returns 400 (why?)
-        #self.title = match1(html_content,
-        #                    r'<h1 title="([^"]+)"')
-
-        # redirect: watchlater
-        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
-            avid = match1(self.url, r'/av(\d+)')
-            p = int(match1(self.url, r'/p(\d+)') or '1')
-            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # redirect: bangumi/play/ss -> bangumi/play/ep
-        # redirect: bangumi.bilibili.com/anime -> bangumi/play/ep
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url) or \
-             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)/play', self.url):
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            ep_id = initial_state['epList'][0]['id']
-            self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # sort it out
-        if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
-            sort = 'audio'
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
-            sort = 'bangumi'
-        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
-            sort = 'bangumi'
-        elif re.match(r'https?://live\.bilibili\.com/', self.url):
-            sort = 'live'
-        elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
-            sort = 'vc'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
-            sort = 'video'
-        else:
-            self.download_playlist_by_url(self.url, **kwargs)
-            return
-
-        # regular av video
-        if sort == 'video':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-
-            playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-
-            # warn if it is a multi-part video
-            pn = initial_state['videoData']['videos']
-            if pn > 1 and not kwargs.get('playlist'):
-                log.w('This is a multipart video. (use --playlist to download all parts.)')
-
-            # set video title
-            self.title = initial_state['videoData']['title']
-            # refine title for a specific part, if it is a multi-part video
-            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
-                    '1')  # use URL to decide p-number, not initial_state['p']
-            if pn > 1:
-                part = initial_state['videoData']['pages'][p - 1]['part']
-                self.title = '%s (P%s. %s)' % (self.title, p, part)
-
-            # construct playinfos
-            avid = initial_state['aid']
-            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
-            current_quality, best_quality = None, None
-            if playinfo is not None:
-                current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
-                if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
-                    best_quality = playinfo['data']['accept_quality'][0]
-            playinfos = []
-            if playinfo is not None:
-                playinfos.append(playinfo)
-            if playinfo_ is not None:
-                playinfos.append(playinfo_)
-            # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
-                # automatic format for durl: qn=0
-                # for dash, qn does not matter
-                if current_quality is None or qn < current_quality:
-                    api_url = self.bilibili_api(avid, cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo = json.loads(api_content)
-                    if api_playinfo['code'] == 0:  # success
-                        playinfos.append(api_playinfo)
-                    else:
-                        message = api_playinfo['data']['message']
-                if best_quality is None or qn <= best_quality:
-                    api_url = self.bilibili_interface_api(cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo_data = json.loads(api_content)
-                    if api_playinfo_data.get('quality'):
-                        playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
-            if not playinfos:
-                log.w(message)
-                # use bilibili error video instead
-                url = 'https://static.hdslb.com/error.mp4'
-                _, container, size = url_info(url)
-                self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
-                return
-
-            for playinfo in playinfos:
-                quality = playinfo['data']['quality']
-                format_id = self.stream_qualities[quality]['id']
-                container = self.stream_qualities[quality]['container'].lower()
-                desc = self.stream_qualities[quality]['desc']
-
-                if 'durl' in playinfo['data']:
-                    src, size = [], 0
-                    for durl in playinfo['data']['durl']:
-                        src.append(durl['url'])
-                        size += durl['size']
-                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-                # DASH formats
-                if 'dash' in playinfo['data']:
-                    audio_size_cache = {}
-                    for video in playinfo['data']['dash']['video']:
-                        # prefer the latter codecs!
-                        s = self.stream_qualities[video['id']]
-                        format_id = 'dash-' + s['id']  # prefix
-                        container = 'mp4'  # enforce MP4 container
-                        desc = s['desc']
-                        audio_quality = s['audio_quality']
-                        baseurl = video['baseUrl']
-                        size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        # find matching audio track
-                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
-                        for audio in playinfo['data']['dash']['audio']:
-                            if int(audio['id']) == audio_quality:
-                                audio_baseurl = audio['baseUrl']
-                                break
-                        if not audio_size_cache.get(audio_quality, False):
-                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-                        size += audio_size_cache[audio_quality]
-
-                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
-
-            # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
-
-        # bangumi
-        elif sort == 'bangumi':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-
-            # warn if this bangumi has more than 1 video
-            epn = len(initial_state['epList'])
-            if epn > 1 and not kwargs.get('playlist'):
-                log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
-
-            # set video title
-            self.title = initial_state['h1Title']
-
-            # construct playinfos
-            ep_id = initial_state['epInfo']['id']
-            avid = initial_state['epInfo']['aid']
-            cid = initial_state['epInfo']['cid']
-            playinfos = []
-            api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            api_playinfo = json.loads(api_content)
-            if api_playinfo['code'] == 0:  # success
-                playinfos.append(api_playinfo)
-            else:
-                log.e(api_playinfo['message'])
-                return
-            current_quality = api_playinfo['result']['quality']
-            # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
-                # automatic format for durl: qn=0
-                # for dash, qn does not matter
-                if qn != current_quality:
-                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo = json.loads(api_content)
-                    if api_playinfo['code'] == 0:  # success
-                        playinfos.append(api_playinfo)
-
-            for playinfo in playinfos:
-                if 'durl' in playinfo['result']:
-                    quality = playinfo['result']['quality']
-                    format_id = self.stream_qualities[quality]['id']
-                    container = self.stream_qualities[quality]['container'].lower()
-                    desc = self.stream_qualities[quality]['desc']
-
-                    src, size = [], 0
-                    for durl in playinfo['result']['durl']:
-                        src.append(durl['url'])
-                        size += durl['size']
-                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-                # DASH formats
-                if 'dash' in playinfo['result']:
-                    for video in playinfo['result']['dash']['video']:
-                        # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
-                        quality = self.height_to_quality(video['height'])  # convert height to quality code
-                        s = self.stream_qualities[quality]
-                        format_id = 'dash-' + s['id']  # prefix
-                        container = 'mp4'  # enforce MP4 container
-                        desc = s['desc']
-                        audio_quality = s['audio_quality']
-                        baseurl = video['baseUrl']
-                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        # find matching audio track
-                        audio_baseurl = playinfo['result']['dash']['audio'][0]['baseUrl']
-                        for audio in playinfo['result']['dash']['audio']:
-                            if int(audio['id']) == audio_quality:
-                                audio_baseurl = audio['baseUrl']
-                                break
-                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
-
-            # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
-
-        # vc video
-        elif sort == 'vc':
-            video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')
-            api_url = self.bilibili_vc_api(video_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            api_playinfo = json.loads(api_content)
-
-            # set video title
-            self.title = '%s (%s)' % (api_playinfo['data']['user']['name'], api_playinfo['data']['item']['id'])
-
-            height = api_playinfo['data']['item']['height']
-            quality = self.height_to_quality(height)  # convert height to quality code
-            s = self.stream_qualities[quality]
-            format_id = s['id']
-            container = 'mp4'  # enforce MP4 container
-            desc = s['desc']
-
-            playurl = api_playinfo['data']['item']['video_playurl']
-            size = int(api_playinfo['data']['item']['video_size'])
-
-            self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
-
-        # live
-        elif sort == 'live':
-            m = re.match(r'https?://live\.bilibili\.com/(\w+)', self.url)
-            short_id = m.group(1)
-            api_url = self.bilibili_live_room_init_api(short_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            room_init_info = json.loads(api_content)
-
-            room_id = room_init_info['data']['room_id']
-            api_url = self.bilibili_live_room_info_api(room_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            room_info = json.loads(api_content)
-
-            # set video title
-            self.title = room_info['data']['title'] + '.' + str(int(time.time()))
-
-            api_url = self.bilibili_live_api(room_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            video_info = json.loads(api_content)
-
-            durls = video_info['data']['durl']
-            playurl = durls[0]['url']
-            container = 'flv'  # enforce FLV container
-            self.streams['flv'] = {'container': container, 'quality': 'unknown',
-                                   'size': 0, 'src': [playurl]}
-
-        # audio
-        elif sort == 'audio':
-            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/au(\d+)', self.url)
-            sid = m.group(1)
-            api_url = self.bilibili_audio_info_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            song_info = json.loads(api_content)
-
-            # set audio title
-            self.title = song_info['data']['title']
-
-            # get lyrics
-            self.lyrics = get_content(song_info['data']['lyric'])
-
-            api_url = self.bilibili_audio_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            audio_info = json.loads(api_content)
-
-            playurl = audio_info['data']['cdns'][0]
-            size = audio_info['data']['size']
-            container = 'mp4'  # enforce MP4 container
-            self.streams['mp4'] = {'container': container,
-                                   'size': size, 'src': [playurl]}
-
-    def extract(self, **kwargs):
-        # set UA and referer for downloading
-        headers = self.bilibili_headers(referer=self.url)
-        self.ua, self.referer = headers['User-Agent'], headers['Referer']
-
-        if not self.streams_sorted:
-            # no stream is available
-            return
-
-        if 'stream_id' in kwargs and kwargs['stream_id']:
-            # extract the stream
-            stream_id = kwargs['stream_id']
-            if stream_id not in self.streams and stream_id not in self.dash_streams:
-                log.e('[Error] Invalid video format.')
-                log.e('Run \'-i\' command with no specific video format to view all available formats.')
-                exit(2)
-        else:
-            # extract stream with the best quality
-            stream_id = self.streams_sorted[0]['id']
-
-    def download_playlist_by_url(self, url, **kwargs):
-        self.url = url
-        kwargs['playlist'] = True
-
-        html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # sort it out
-        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
-            sort = 'bangumi'
-        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
-            sort = 'bangumi'
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
-            re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
-            sort = 'bangumi_md'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
-            sort = 'video'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
-            sort = 'space_channel'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
-            sort = 'space_favlist'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
-            sort = 'space_video'
-        elif re.match(r'https?://(www\.)?bilibili\.com/audio/am(\d+)', self.url):
-            sort = 'audio_menu'
-        else:
-            log.e('[Error] Unsupported URL pattern.')
-            exit(1)
-
-        # regular av video
-        if sort == 'video':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            aid = initial_state['videoData']['aid']
-            pn = initial_state['videoData']['videos']
-            for pi in range(1, pn + 1):
-                purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
-                self.__class__().download_by_url(purl, **kwargs)
-
-        elif sort == 'bangumi':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            epn, i = len(initial_state['epList']), 0
-            for ep in initial_state['epList']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                ep_id = ep['id']
-                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
-                self.__class__().download_by_url(epurl, **kwargs)
-
-        elif sort == 'bangumi_md':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            epn, i = len(initial_state['mediaInfo']['episodes']), 0
-            for ep in initial_state['mediaInfo']['episodes']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                ep_id = ep['ep_id']
-                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
-                self.__class__().download_by_url(epurl, **kwargs)
-
-        elif sort == 'space_channel':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
-            mid, cid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_channel_api(mid, cid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            channel_info = json.loads(api_content)
-            # TBD: channel of more than 100 videos
-
-            epn, i = len(channel_info['data']['list']['archives']), 0
-            for video in channel_info['data']['list']['archives']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'space_favlist':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
-            vmid, fid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_favlist_api(vmid, fid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            favlist_info = json.loads(api_content)
-            pc = favlist_info['data']['pagecount']
-
-            for pn in range(1, pc + 1):
-                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
-                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-                favlist_info = json.loads(api_content)
-
-                epn, i = len(favlist_info['data']['archives']), 0
-                for video in favlist_info['data']['archives']:
-                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                    self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'space_video':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
-            mid = m.group(1)
-            api_url = self.bilibili_space_video_api(mid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            videos_info = json.loads(api_content)
-            pc = videos_info['data']['pages']
-
-            for pn in range(1, pc + 1):
-                api_url = self.bilibili_space_video_api(mid, pn=pn)
-                api_content = get_content(api_url, headers=self.bilibili_headers())
-                videos_info = json.loads(api_content)
-
-                epn, i = len(videos_info['data']['vlist']), 0
-                for video in videos_info['data']['vlist']:
-                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                    self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'audio_menu':
-            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
-            sid = m.group(1)
-            #api_url = self.bilibili_audio_menu_info_api(sid)
-            #api_content = get_content(api_url, headers=self.bilibili_headers())
-            #menu_info = json.loads(api_content)
-            api_url = self.bilibili_audio_menu_song_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            menusong_info = json.loads(api_content)
-            epn, i = len(menusong_info['data']['data']), 0
-            for song in menusong_info['data']['data']:
-                i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
-                url = 'https://www.bilibili.com/audio/au%s' % song['id']
-                self.__class__().download_by_url(url, **kwargs)
-
-
-site = Bilibili()
-download = site.download_by_url
-download_playlist = site.download_playlist_by_url
-
-bilibili_download = download
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+import hashlib
+
+class Bilibili(VideoExtractor):
+    name = "Bilibili"
+
+    # Bilibili media encoding options, in descending quality order.
+    stream_types = [
+        {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
+        {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
+        {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
+        {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
+        {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
+        {'id': 'hdmp4', 'quality': 48, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P (MP4)'},
+        {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
+        {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
+         'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
+        # 'quality': 15?
+        {'id': 'mp4', 'quality': 0},
+    ]
+
+    @staticmethod
+    def height_to_quality(height):
+        if height <= 360:
+            return 16
+        elif height <= 480:
+            return 32
+        elif height <= 720:
+            return 64
+        else:
+            return 80
+
+    @staticmethod
+    def bilibili_headers(referer=None, cookie=None):
+        # a reasonable UA
+        ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
+        headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
+        if referer is not None:
+            headers.update({'Referer': referer})
+        if cookie is not None:
+            headers.update({'Cookie': cookie})
+        return headers
+
+    @staticmethod
+    def bilibili_api(avid, cid, qn=0):
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
+
+    @staticmethod
+    def bilibili_audio_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/url?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_menu_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/menu/info?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_menu_song_api(sid, ps=100):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
+
+    @staticmethod
+    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
+        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
+
+    @staticmethod
+    def bilibili_interface_api(cid, qn=0):
+        entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
+        appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
+        params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
+        chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
+        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+
+    @staticmethod
+    def bilibili_live_api(cid):
+        return 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid=%s&quality=0&platform=web' % cid
+
+    @staticmethod
+    def bilibili_live_room_info_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id=%s' % room_id
+
+    @staticmethod
+    def bilibili_live_room_init_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
+
+    @staticmethod
+    def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
+
+    @staticmethod
+    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
+
+    @staticmethod
+    def bilibili_space_video_api(mid, pn=1, ps=100):
+        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
+
+    @staticmethod
+    def bilibili_vc_api(video_id):
+        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
+
+    @staticmethod
+    def url_size(url, faker=False, headers={},err_value=0):
+        try:
+            return url_size(url,faker,headers)
+        except:
+            return err_value
+
+    def prepare(self, **kwargs):
+        self.stream_qualities = {s['quality']: s for s in self.stream_types}
+
+        try:
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+        except:
+            html_content = ''  # live always returns 400 (why?)
+        #self.title = match1(html_content,
+        #                    r'<h1 title="([^"]+)"')
+
+        # redirect: watchlater
+        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
+            avid = match1(self.url, r'/av(\d+)')
+            p = int(match1(self.url, r'/p(\d+)') or '1')
+            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # redirect: bangumi/play/ss -> bangumi/play/ep
+        # redirect: bangumi.bilibili.com/anime -> bangumi/play/ep
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url) or \
+             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)/play', self.url):
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            ep_id = initial_state['epList'][0]['id']
+            self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # sort it out
+        if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
+            sort = 'audio'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://live\.bilibili\.com/', self.url):
+            sort = 'live'
+        elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
+            sort = 'vc'
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+        else:
+            self.download_playlist_by_url(self.url, **kwargs)
+            return
+
+        # regular av video
+        if sort == 'video':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+
+            playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+
+            # warn if it is a multi-part video
+            pn = initial_state['videoData']['videos']
+            if pn > 1 and not kwargs.get('playlist'):
+                log.w('This is a multipart video. (use --playlist to download all parts.)')
+
+            # set video title
+            self.title = initial_state['videoData']['title']
+            # refine title for a specific part, if it is a multi-part video
+            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
+                    '1')  # use URL to decide p-number, not initial_state['p']
+            if pn > 1:
+                part = initial_state['videoData']['pages'][p - 1]['part']
+                self.title = '%s (P%s. %s)' % (self.title, p, part)
+
+            # construct playinfos
+            avid = initial_state['aid']
+            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
+            current_quality, best_quality = None, None
+            if playinfo is not None:
+                current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
+                if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
+                    best_quality = playinfo['data']['accept_quality'][0]
+            playinfos = []
+            if playinfo is not None:
+                playinfos.append(playinfo)
+            if playinfo_ is not None:
+                playinfos.append(playinfo_)
+            # get alternative formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # for dash, qn does not matter
+                if current_quality is None or qn < current_quality:
+                    api_url = self.bilibili_api(avid, cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
+                    else:
+                        message = api_playinfo['data']['message']
+                if best_quality is None or qn <= best_quality:
+                    api_url = self.bilibili_interface_api(cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo_data = json.loads(api_content)
+                    if api_playinfo_data.get('quality'):
+                        playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
+            if not playinfos:
+                log.w(message)
+                # use bilibili error video instead
+                url = 'https://static.hdslb.com/error.mp4'
+                _, container, size = url_info(url)
+                self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
+                return
+
+            for playinfo in playinfos:
+                quality = playinfo['data']['quality']
+                format_id = self.stream_qualities[quality]['id']
+                container = self.stream_qualities[quality]['container'].lower()
+                desc = self.stream_qualities[quality]['desc']
+
+                if 'durl' in playinfo['data']:
+                    src, size = [], 0
+                    for durl in playinfo['data']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['data']:
+                    audio_size_cache = {}
+                    for video in playinfo['data']['dash']['video']:
+                        # prefer the latter codecs!
+                        s = self.stream_qualities[video['id']]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['data']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        if not audio_size_cache.get(audio_quality, False):
+                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                        size += audio_size_cache[audio_quality]
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+        # bangumi
+        elif sort == 'bangumi':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+
+            # warn if this bangumi has more than 1 video
+            epn = len(initial_state['epList'])
+            if epn > 1 and not kwargs.get('playlist'):
+                log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
+
+            # set video title
+            self.title = initial_state['h1Title']
+
+            # construct playinfos
+            ep_id = initial_state['epInfo']['id']
+            avid = initial_state['epInfo']['aid']
+            cid = initial_state['epInfo']['cid']
+            playinfos = []
+            api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_playinfo = json.loads(api_content)
+            if api_playinfo['code'] == 0:  # success
+                playinfos.append(api_playinfo)
+            else:
+                log.e(api_playinfo['message'])
+                return
+            current_quality = api_playinfo['result']['quality']
+            # get alternative formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # for dash, qn does not matter
+                if qn != current_quality:
+                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
+
+            for playinfo in playinfos:
+                if 'durl' in playinfo['result']:
+                    quality = playinfo['result']['quality']
+                    format_id = self.stream_qualities[quality]['id']
+                    container = self.stream_qualities[quality]['container'].lower()
+                    desc = self.stream_qualities[quality]['desc']
+
+                    src, size = [], 0
+                    for durl in playinfo['result']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['result']:
+                    for video in playinfo['result']['dash']['video']:
+                        # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
+                        quality = self.height_to_quality(video['height'])  # convert height to quality code
+                        s = self.stream_qualities[quality]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['result']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['result']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+        # vc video
+        elif sort == 'vc':
+            video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')
+            api_url = self.bilibili_vc_api(video_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_playinfo = json.loads(api_content)
+
+            # set video title
+            self.title = '%s (%s)' % (api_playinfo['data']['user']['name'], api_playinfo['data']['item']['id'])
+
+            height = api_playinfo['data']['item']['height']
+            quality = self.height_to_quality(height)  # convert height to quality code
+            s = self.stream_qualities[quality]
+            format_id = s['id']
+            container = 'mp4'  # enforce MP4 container
+            desc = s['desc']
+
+            playurl = api_playinfo['data']['item']['video_playurl']
+            size = int(api_playinfo['data']['item']['video_size'])
+
+            self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
+
+        # live
+        elif sort == 'live':
+            m = re.match(r'https?://live\.bilibili\.com/(\w+)', self.url)
+            short_id = m.group(1)
+            api_url = self.bilibili_live_room_init_api(short_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_init_info = json.loads(api_content)
+
+            room_id = room_init_info['data']['room_id']
+            api_url = self.bilibili_live_room_info_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_info = json.loads(api_content)
+
+            # set video title
+            self.title = room_info['data']['title'] + '.' + str(int(time.time()))
+
+            api_url = self.bilibili_live_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            video_info = json.loads(api_content)
+
+            durls = video_info['data']['durl']
+            playurl = durls[0]['url']
+            container = 'flv'  # enforce FLV container
+            self.streams['flv'] = {'container': container, 'quality': 'unknown',
+                                   'size': 0, 'src': [playurl]}
+
+        # audio
+        elif sort == 'audio':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/au(\d+)', self.url)
+            sid = m.group(1)
+            api_url = self.bilibili_audio_info_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            song_info = json.loads(api_content)
+
+            # set audio title
+            self.title = song_info['data']['title']
+
+            # get lyrics
+            self.lyrics = get_content(song_info['data']['lyric'])
+
+            api_url = self.bilibili_audio_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            audio_info = json.loads(api_content)
+
+            playurl = audio_info['data']['cdns'][0]
+            size = audio_info['data']['size']
+            container = 'mp4'  # enforce MP4 container
+            self.streams['mp4'] = {'container': container,
+                                   'size': size, 'src': [playurl]}
+
+
+    def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
+        #response for interaction video
+        #主要针对互动视频，使用cid而不是url来相互区分

+        self.stream_qualities = {s['quality']: s for s in self.stream_types}
+        self.title = title
+        self.url = url
+
+        #try:
+        #    html_content = get_content(self.url, headers=self.bilibili_headers())
+        #except:
+        #    html_content = ''
+
+        #initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+        #initial_state = json.loads(initial_state_text)
+
+        # playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+        # playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+        # html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+        # playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+        # playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+        current_quality, best_quality = None, None
+        if playinfo is not None:
+            current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
+            if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
+                best_quality = playinfo['data']['accept_quality'][0]
+        playinfos = []
+        if playinfo is not None:
+            playinfos.append(playinfo)
+        if playinfo_ is not None:
+            playinfos.append(playinfo_)
+        # get alternative formats from API
+        for qn in [80, 64, 32, 16]:
+            # automatic format for durl: qn=0
+            # for dash, qn does not matter
+            if current_quality is None or qn < current_quality:
+                api_url = self.bilibili_api(avid, cid, qn=qn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                api_playinfo = json.loads(api_content)
+                if api_playinfo['code'] == 0:  # success
+                    playinfos.append(api_playinfo)
+                else:
+                    message = api_playinfo['data']['message']
+            if best_quality is None or qn <= best_quality:
+                api_url = self.bilibili_interface_api(cid, qn=qn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                api_playinfo_data = json.loads(api_content)
+                if api_playinfo_data.get('quality'):
+                    playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
+        if not playinfos:
+            log.w(message)
+            # use bilibili error video instead
+            url = 'https://static.hdslb.com/error.mp4'
+            _, container, size = url_info(url)
+            self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
+            return
+
+        for playinfo in playinfos:
+            quality = playinfo['data']['quality']
+            format_id = self.stream_qualities[quality]['id']
+            container = self.stream_qualities[quality]['container'].lower()
+            desc = self.stream_qualities[quality]['desc']
+
+            if 'durl' in playinfo['data']:
+                src, size = [], 0
+                for durl in playinfo['data']['durl']:
+                    src.append(durl['url'])
+                    size += durl['size']
+                self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+            # DASH formats
+            if 'dash' in playinfo['data']:
+                audio_size_cache = {}
+                for video in playinfo['data']['dash']['video']:
+                    # prefer the latter codecs!
+                    s = self.stream_qualities[video['id']]
+                    format_id = 'dash-' + s['id']  # prefix
+                    container = 'mp4'  # enforce MP4 container
+                    desc = s['desc']
+                    audio_quality = s['audio_quality']
+                    baseurl = video['baseUrl']
+                    size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    # find matching audio track
+                    audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                    for audio in playinfo['data']['dash']['audio']:
+                        if int(audio['id']) == audio_quality:
+                            audio_baseurl = audio['baseUrl']
+                            break
+                    if not audio_size_cache.get(audio_quality, False):
+                        audio_size_cache[audio_quality] = self.url_size(audio_baseurl,
+                                                                        headers=self.bilibili_headers(referer=self.url))
+                    size += audio_size_cache[audio_quality]
+
+                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+        # get danmaku
+        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+    def extract(self, **kwargs):
+        # set UA and referer for downloading
+        headers = self.bilibili_headers(referer=self.url)
+        self.ua, self.referer = headers['User-Agent'], headers['Referer']
+
+        if not self.streams_sorted:
+            # no stream is available
+            return
+
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # extract the stream
+            stream_id = kwargs['stream_id']
+            if stream_id not in self.streams and stream_id not in self.dash_streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+
+    def download_playlist_by_url(self, url, **kwargs):
+        self.url = url
+        kwargs['playlist'] = True
+
+        html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # sort it out
+        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
+            re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
+            sort = 'bangumi_md'
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
+            sort = 'space_channel'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
+            sort = 'space_favlist'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
+            sort = 'space_video'
+        elif re.match(r'https?://(www\.)?bilibili\.com/audio/am(\d+)', self.url):
+            sort = 'audio_menu'
+        else:
+            log.e('[Error] Unsupported URL pattern.')
+            exit(1)
+
+        # regular av video
+        if sort == 'video':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            aid = initial_state['videoData']['aid']
+            pn = initial_state['videoData']['videos']
+            if pn!= len(initial_state['videoData']['pages']):#interaction video 互动视频
+                search_node_list = []
+                download_cid_set = set([initial_state['videoData']['cid']])
+                node_infos = {}
+                params = {
+                        'id': 'cid:{}'.format(initial_state['videoData']['cid']),
+                        'aid': str(aid)
+                }
+                urlcontent = get_content('https://api.bilibili.com/x/player.so?'+parse.urlencode(params), headers=self.bilibili_headers(referer='https://www.bilibili.com/video/av{}'.format(aid)))
+                graph_version = json.loads(urlcontent[urlcontent.find('<interaction>')+13:urlcontent.find('</interaction>')])['graph_version']
+                params = {
+                    'aid': str(aid),
+                    'graph_version': graph_version,
+                    'platform': 'pc',
+                    'portal': 0,
+                    'screen': 0,
+                }
+                node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
+                node_infos.update({1: (initial_state['videoData']['cid'], node_info['data']['title'])})
+
+                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+
+                self.prepare_by_cid(aid, initial_state['videoData']['cid'], initial_state['videoData']['title'] + ('P{}. {}'.format(1, node_info['data']['title'])),html_content,playinfo,playinfo_,url)
+                self.extract(**kwargs)
+                self.download(**kwargs)
+                for choice in node_info['data']['edges']['choices']:
+                    search_node_list.append(choice['node_id'])
+                    if not choice['cid'] in download_cid_set:
+                        download_cid_set.add(choice['cid'])
+                        node_infos.update({len(download_cid_set): (choice['cid'], choice['option'])})
+                        self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
+                        self.extract(**kwargs)
+                        self.download(**kwargs)
+                while len(search_node_list)>0:
+                    node_id = search_node_list.pop(0)
+                    params.update({'node_id':node_id})
+                    node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
+                    if node_info['data'].__contains__('edges'):
+                        for choice in node_info['data']['edges']['choices']:
+                            search_node_list.append(choice['node_id'])
+                            if not choice['cid'] in download_cid_set:
+                                download_cid_set.add(choice['cid'] )
+                                node_infos.update({len(download_cid_set):(choice['cid'],choice['option'])})
+                                self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
+                                self.extract(**kwargs)
+                                self.download(**kwargs)
+            else:
+                for pi in range(1, pn + 1):
+                    purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
+                    self.__class__().download_by_url(purl, **kwargs)
+
+        elif sort == 'bangumi':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['epList']), 0
+            for ep in initial_state['epList']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                ep_id = ep['id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+
+        elif sort == 'bangumi_md':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['mediaInfo']['episodes']), 0
+            for ep in initial_state['mediaInfo']['episodes']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                ep_id = ep['ep_id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+
+        elif sort == 'space_channel':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
+            mid, cid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_channel_api(mid, cid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            channel_info = json.loads(api_content)
+            # TBD: channel of more than 100 videos
+
+            epn, i = len(channel_info['data']['list']['archives']), 0
+            for video in channel_info['data']['list']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'space_favlist':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
+            vmid, fid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_favlist_api(vmid, fid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            favlist_info = json.loads(api_content)
+            pc = favlist_info['data']['pagecount']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                favlist_info = json.loads(api_content)
+
+                epn, i = len(favlist_info['data']['archives']), 0
+                for video in favlist_info['data']['archives']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'space_video':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
+            mid = m.group(1)
+            api_url = self.bilibili_space_video_api(mid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            videos_info = json.loads(api_content)
+            pc = videos_info['data']['pages']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_video_api(mid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                videos_info = json.loads(api_content)
+
+                epn, i = len(videos_info['data']['vlist']), 0
+                for video in videos_info['data']['vlist']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'audio_menu':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
+            sid = m.group(1)
+            #api_url = self.bilibili_audio_menu_info_api(sid)
+            #api_content = get_content(api_url, headers=self.bilibili_headers())
+            #menu_info = json.loads(api_content)
+            api_url = self.bilibili_audio_menu_song_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            menusong_info = json.loads(api_content)
+            epn, i = len(menusong_info['data']['data']), 0
+            for song in menusong_info['data']['data']:
+                i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
+                url = 'https://www.bilibili.com/audio/au%s' % song['id']
+                self.__class__().download_by_url(url, **kwargs)
+
+
+site = Bilibili()
+download = site.download_by_url
+download_playlist = site.download_playlist_by_url
+
+bilibili_download = download

From da62c488cd61d11adf29b8245ed132e62584f629 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Wed, 2 Oct 2019 18:37:06 +0800
Subject: [PATCH 0892/1225] Update bilibili.py

---
 src/you_get/extractors/bilibili.py | 1435 ++++++++++++++--------------
 1 file changed, 709 insertions(+), 726 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 25cc0df3a7..7eaf9b9fe0 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -1,726 +1,709 @@
-#!/usr/bin/env python
-
-from ..common import *
-from ..extractor import VideoExtractor
-
-import hashlib
-
-class Bilibili(VideoExtractor):
-    name = "Bilibili"
-
-    # Bilibili media encoding options, in descending quality order.
-    stream_types = [
-        {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
-        {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
-        {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
-        {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
-        {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
-        {'id': 'hdmp4', 'quality': 48, 'audio_quality': 30280,
-         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P (MP4)'},
-        {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
-        {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
-         'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
-        # 'quality': 15?
-        {'id': 'mp4', 'quality': 0},
-    ]
-
-    @staticmethod
-    def height_to_quality(height):
-        if height <= 360:
-            return 16
-        elif height <= 480:
-            return 32
-        elif height <= 720:
-            return 64
-        else:
-            return 80
-
-    @staticmethod
-    def bilibili_headers(referer=None, cookie=None):
-        # a reasonable UA
-        ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
-        headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
-        if referer is not None:
-            headers.update({'Referer': referer})
-        if cookie is not None:
-            headers.update({'Cookie': cookie})
-        return headers
-
-    @staticmethod
-    def bilibili_api(avid, cid, qn=0):
-        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
-
-    @staticmethod
-    def bilibili_audio_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/url?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_info_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_menu_info_api(sid):
-        return 'https://www.bilibili.com/audio/music-service-c/web/menu/info?sid=%s' % sid
-
-    @staticmethod
-    def bilibili_audio_menu_song_api(sid, ps=100):
-        return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
-
-    @staticmethod
-    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
-        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
-
-    @staticmethod
-    def bilibili_interface_api(cid, qn=0):
-        entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
-        appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
-        params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
-        chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
-        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
-
-    @staticmethod
-    def bilibili_live_api(cid):
-        return 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid=%s&quality=0&platform=web' % cid
-
-    @staticmethod
-    def bilibili_live_room_info_api(room_id):
-        return 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id=%s' % room_id
-
-    @staticmethod
-    def bilibili_live_room_init_api(room_id):
-        return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
-
-    @staticmethod
-    def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
-        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
-
-    @staticmethod
-    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
-        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
-
-    @staticmethod
-    def bilibili_space_video_api(mid, pn=1, ps=100):
-        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
-
-    @staticmethod
-    def bilibili_vc_api(video_id):
-        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
-
-    @staticmethod
-    def url_size(url, faker=False, headers={},err_value=0):
-        try:
-            return url_size(url,faker,headers)
-        except:
-            return err_value
-
-    def prepare(self, **kwargs):
-        self.stream_qualities = {s['quality']: s for s in self.stream_types}
-
-        try:
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-        except:
-            html_content = ''  # live always returns 400 (why?)
-        #self.title = match1(html_content,
-        #                    r'<h1 title="([^"]+)"')
-
-        # redirect: watchlater
-        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
-            avid = match1(self.url, r'/av(\d+)')
-            p = int(match1(self.url, r'/p(\d+)') or '1')
-            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # redirect: bangumi/play/ss -> bangumi/play/ep
-        # redirect: bangumi.bilibili.com/anime -> bangumi/play/ep
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url) or \
-             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)/play', self.url):
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            ep_id = initial_state['epList'][0]['id']
-            self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
-            html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # sort it out
-        if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
-            sort = 'audio'
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
-            sort = 'bangumi'
-        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
-            sort = 'bangumi'
-        elif re.match(r'https?://live\.bilibili\.com/', self.url):
-            sort = 'live'
-        elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
-            sort = 'vc'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
-            sort = 'video'
-        else:
-            self.download_playlist_by_url(self.url, **kwargs)
-            return
-
-        # regular av video
-        if sort == 'video':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-
-            playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-            playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-
-            # warn if it is a multi-part video
-            pn = initial_state['videoData']['videos']
-            if pn > 1 and not kwargs.get('playlist'):
-                log.w('This is a multipart video. (use --playlist to download all parts.)')
-
-            # set video title
-            self.title = initial_state['videoData']['title']
-            # refine title for a specific part, if it is a multi-part video
-            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
-                    '1')  # use URL to decide p-number, not initial_state['p']
-            if pn > 1:
-                part = initial_state['videoData']['pages'][p - 1]['part']
-                self.title = '%s (P%s. %s)' % (self.title, p, part)
-
-            # construct playinfos
-            avid = initial_state['aid']
-            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
-            current_quality, best_quality = None, None
-            if playinfo is not None:
-                current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
-                if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
-                    best_quality = playinfo['data']['accept_quality'][0]
-            playinfos = []
-            if playinfo is not None:
-                playinfos.append(playinfo)
-            if playinfo_ is not None:
-                playinfos.append(playinfo_)
-            # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
-                # automatic format for durl: qn=0
-                # for dash, qn does not matter
-                if current_quality is None or qn < current_quality:
-                    api_url = self.bilibili_api(avid, cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo = json.loads(api_content)
-                    if api_playinfo['code'] == 0:  # success
-                        playinfos.append(api_playinfo)
-                    else:
-                        message = api_playinfo['data']['message']
-                if best_quality is None or qn <= best_quality:
-                    api_url = self.bilibili_interface_api(cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo_data = json.loads(api_content)
-                    if api_playinfo_data.get('quality'):
-                        playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
-            if not playinfos:
-                log.w(message)
-                # use bilibili error video instead
-                url = 'https://static.hdslb.com/error.mp4'
-                _, container, size = url_info(url)
-                self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
-                return
-
-            for playinfo in playinfos:
-                quality = playinfo['data']['quality']
-                format_id = self.stream_qualities[quality]['id']
-                container = self.stream_qualities[quality]['container'].lower()
-                desc = self.stream_qualities[quality]['desc']
-
-                if 'durl' in playinfo['data']:
-                    src, size = [], 0
-                    for durl in playinfo['data']['durl']:
-                        src.append(durl['url'])
-                        size += durl['size']
-                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-                # DASH formats
-                if 'dash' in playinfo['data']:
-                    audio_size_cache = {}
-                    for video in playinfo['data']['dash']['video']:
-                        # prefer the latter codecs!
-                        s = self.stream_qualities[video['id']]
-                        format_id = 'dash-' + s['id']  # prefix
-                        container = 'mp4'  # enforce MP4 container
-                        desc = s['desc']
-                        audio_quality = s['audio_quality']
-                        baseurl = video['baseUrl']
-                        size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        # find matching audio track
-                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
-                        for audio in playinfo['data']['dash']['audio']:
-                            if int(audio['id']) == audio_quality:
-                                audio_baseurl = audio['baseUrl']
-                                break
-                        if not audio_size_cache.get(audio_quality, False):
-                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-                        size += audio_size_cache[audio_quality]
-
-                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
-
-            # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
-
-        # bangumi
-        elif sort == 'bangumi':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-
-            # warn if this bangumi has more than 1 video
-            epn = len(initial_state['epList'])
-            if epn > 1 and not kwargs.get('playlist'):
-                log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
-
-            # set video title
-            self.title = initial_state['h1Title']
-
-            # construct playinfos
-            ep_id = initial_state['epInfo']['id']
-            avid = initial_state['epInfo']['aid']
-            cid = initial_state['epInfo']['cid']
-            playinfos = []
-            api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            api_playinfo = json.loads(api_content)
-            if api_playinfo['code'] == 0:  # success
-                playinfos.append(api_playinfo)
-            else:
-                log.e(api_playinfo['message'])
-                return
-            current_quality = api_playinfo['result']['quality']
-            # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
-                # automatic format for durl: qn=0
-                # for dash, qn does not matter
-                if qn != current_quality:
-                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
-                    api_playinfo = json.loads(api_content)
-                    if api_playinfo['code'] == 0:  # success
-                        playinfos.append(api_playinfo)
-
-            for playinfo in playinfos:
-                if 'durl' in playinfo['result']:
-                    quality = playinfo['result']['quality']
-                    format_id = self.stream_qualities[quality]['id']
-                    container = self.stream_qualities[quality]['container'].lower()
-                    desc = self.stream_qualities[quality]['desc']
-
-                    src, size = [], 0
-                    for durl in playinfo['result']['durl']:
-                        src.append(durl['url'])
-                        size += durl['size']
-                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-                # DASH formats
-                if 'dash' in playinfo['result']:
-                    for video in playinfo['result']['dash']['video']:
-                        # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
-                        quality = self.height_to_quality(video['height'])  # convert height to quality code
-                        s = self.stream_qualities[quality]
-                        format_id = 'dash-' + s['id']  # prefix
-                        container = 'mp4'  # enforce MP4 container
-                        desc = s['desc']
-                        audio_quality = s['audio_quality']
-                        baseurl = video['baseUrl']
-                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        # find matching audio track
-                        audio_baseurl = playinfo['result']['dash']['audio'][0]['baseUrl']
-                        for audio in playinfo['result']['dash']['audio']:
-                            if int(audio['id']) == audio_quality:
-                                audio_baseurl = audio['baseUrl']
-                                break
-                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
-
-            # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
-
-        # vc video
-        elif sort == 'vc':
-            video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')
-            api_url = self.bilibili_vc_api(video_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            api_playinfo = json.loads(api_content)
-
-            # set video title
-            self.title = '%s (%s)' % (api_playinfo['data']['user']['name'], api_playinfo['data']['item']['id'])
-
-            height = api_playinfo['data']['item']['height']
-            quality = self.height_to_quality(height)  # convert height to quality code
-            s = self.stream_qualities[quality]
-            format_id = s['id']
-            container = 'mp4'  # enforce MP4 container
-            desc = s['desc']
-
-            playurl = api_playinfo['data']['item']['video_playurl']
-            size = int(api_playinfo['data']['item']['video_size'])
-
-            self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
-
-        # live
-        elif sort == 'live':
-            m = re.match(r'https?://live\.bilibili\.com/(\w+)', self.url)
-            short_id = m.group(1)
-            api_url = self.bilibili_live_room_init_api(short_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            room_init_info = json.loads(api_content)
-
-            room_id = room_init_info['data']['room_id']
-            api_url = self.bilibili_live_room_info_api(room_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            room_info = json.loads(api_content)
-
-            # set video title
-            self.title = room_info['data']['title'] + '.' + str(int(time.time()))
-
-            api_url = self.bilibili_live_api(room_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            video_info = json.loads(api_content)
-
-            durls = video_info['data']['durl']
-            playurl = durls[0]['url']
-            container = 'flv'  # enforce FLV container
-            self.streams['flv'] = {'container': container, 'quality': 'unknown',
-                                   'size': 0, 'src': [playurl]}
-
-        # audio
-        elif sort == 'audio':
-            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/au(\d+)', self.url)
-            sid = m.group(1)
-            api_url = self.bilibili_audio_info_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            song_info = json.loads(api_content)
-
-            # set audio title
-            self.title = song_info['data']['title']
-
-            # get lyrics
-            self.lyrics = get_content(song_info['data']['lyric'])
-
-            api_url = self.bilibili_audio_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            audio_info = json.loads(api_content)
-
-            playurl = audio_info['data']['cdns'][0]
-            size = audio_info['data']['size']
-            container = 'mp4'  # enforce MP4 container
-            self.streams['mp4'] = {'container': container,
-                                   'size': size, 'src': [playurl]}
-
-
-    def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
-        #response for interaction video
-        #主要针对互动视频，使用cid而不是url来相互区分

-        self.stream_qualities = {s['quality']: s for s in self.stream_types}
-        self.title = title
-        self.url = url
-
-        #try:
-        #    html_content = get_content(self.url, headers=self.bilibili_headers())
-        #except:
-        #    html_content = ''
-
-        #initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-        #initial_state = json.loads(initial_state_text)
-
-        # playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-        # playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-        # html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-        # playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-        # playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-        current_quality, best_quality = None, None
-        if playinfo is not None:
-            current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
-            if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
-                best_quality = playinfo['data']['accept_quality'][0]
-        playinfos = []
-        if playinfo is not None:
-            playinfos.append(playinfo)
-        if playinfo_ is not None:
-            playinfos.append(playinfo_)
-        # get alternative formats from API
-        for qn in [80, 64, 32, 16]:
-            # automatic format for durl: qn=0
-            # for dash, qn does not matter
-            if current_quality is None or qn < current_quality:
-                api_url = self.bilibili_api(avid, cid, qn=qn)
-                api_content = get_content(api_url, headers=self.bilibili_headers())
-                api_playinfo = json.loads(api_content)
-                if api_playinfo['code'] == 0:  # success
-                    playinfos.append(api_playinfo)
-                else:
-                    message = api_playinfo['data']['message']
-            if best_quality is None or qn <= best_quality:
-                api_url = self.bilibili_interface_api(cid, qn=qn)
-                api_content = get_content(api_url, headers=self.bilibili_headers())
-                api_playinfo_data = json.loads(api_content)
-                if api_playinfo_data.get('quality'):
-                    playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
-        if not playinfos:
-            log.w(message)
-            # use bilibili error video instead
-            url = 'https://static.hdslb.com/error.mp4'
-            _, container, size = url_info(url)
-            self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
-            return
-
-        for playinfo in playinfos:
-            quality = playinfo['data']['quality']
-            format_id = self.stream_qualities[quality]['id']
-            container = self.stream_qualities[quality]['container'].lower()
-            desc = self.stream_qualities[quality]['desc']
-
-            if 'durl' in playinfo['data']:
-                src, size = [], 0
-                for durl in playinfo['data']['durl']:
-                    src.append(durl['url'])
-                    size += durl['size']
-                self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
-
-            # DASH formats
-            if 'dash' in playinfo['data']:
-                audio_size_cache = {}
-                for video in playinfo['data']['dash']['video']:
-                    # prefer the latter codecs!
-                    s = self.stream_qualities[video['id']]
-                    format_id = 'dash-' + s['id']  # prefix
-                    container = 'mp4'  # enforce MP4 container
-                    desc = s['desc']
-                    audio_quality = s['audio_quality']
-                    baseurl = video['baseUrl']
-                    size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
-
-                    # find matching audio track
-                    audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
-                    for audio in playinfo['data']['dash']['audio']:
-                        if int(audio['id']) == audio_quality:
-                            audio_baseurl = audio['baseUrl']
-                            break
-                    if not audio_size_cache.get(audio_quality, False):
-                        audio_size_cache[audio_quality] = self.url_size(audio_baseurl,
-                                                                        headers=self.bilibili_headers(referer=self.url))
-                    size += audio_size_cache[audio_quality]
-
-                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
-
-        # get danmaku
-        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
-
-    def extract(self, **kwargs):
-        # set UA and referer for downloading
-        headers = self.bilibili_headers(referer=self.url)
-        self.ua, self.referer = headers['User-Agent'], headers['Referer']
-
-        if not self.streams_sorted:
-            # no stream is available
-            return
-
-        if 'stream_id' in kwargs and kwargs['stream_id']:
-            # extract the stream
-            stream_id = kwargs['stream_id']
-            if stream_id not in self.streams and stream_id not in self.dash_streams:
-                log.e('[Error] Invalid video format.')
-                log.e('Run \'-i\' command with no specific video format to view all available formats.')
-                exit(2)
-        else:
-            # extract stream with the best quality
-            stream_id = self.streams_sorted[0]['id']
-
-    def download_playlist_by_url(self, url, **kwargs):
-        self.url = url
-        kwargs['playlist'] = True
-
-        html_content = get_content(self.url, headers=self.bilibili_headers())
-
-        # sort it out
-        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
-            sort = 'bangumi'
-        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
-            sort = 'bangumi'
-        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
-            re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
-            sort = 'bangumi_md'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
-            sort = 'video'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
-            sort = 'space_channel'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
-            sort = 'space_favlist'
-        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
-            sort = 'space_video'
-        elif re.match(r'https?://(www\.)?bilibili\.com/audio/am(\d+)', self.url):
-            sort = 'audio_menu'
-        else:
-            log.e('[Error] Unsupported URL pattern.')
-            exit(1)
-
-        # regular av video
-        if sort == 'video':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            aid = initial_state['videoData']['aid']
-            pn = initial_state['videoData']['videos']
-            if pn!= len(initial_state['videoData']['pages']):#interaction video 互动视频
-                search_node_list = []
-                download_cid_set = set([initial_state['videoData']['cid']])
-                node_infos = {}
-                params = {
-                        'id': 'cid:{}'.format(initial_state['videoData']['cid']),
-                        'aid': str(aid)
-                }
-                urlcontent = get_content('https://api.bilibili.com/x/player.so?'+parse.urlencode(params), headers=self.bilibili_headers(referer='https://www.bilibili.com/video/av{}'.format(aid)))
-                graph_version = json.loads(urlcontent[urlcontent.find('<interaction>')+13:urlcontent.find('</interaction>')])['graph_version']
-                params = {
-                    'aid': str(aid),
-                    'graph_version': graph_version,
-                    'platform': 'pc',
-                    'portal': 0,
-                    'screen': 0,
-                }
-                node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
-                node_infos.update({1: (initial_state['videoData']['cid'], node_info['data']['title'])})
-
-                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-
-                self.prepare_by_cid(aid, initial_state['videoData']['cid'], initial_state['videoData']['title'] + ('P{}. {}'.format(1, node_info['data']['title'])),html_content,playinfo,playinfo_,url)
-                self.extract(**kwargs)
-                self.download(**kwargs)
-                for choice in node_info['data']['edges']['choices']:
-                    search_node_list.append(choice['node_id'])
-                    if not choice['cid'] in download_cid_set:
-                        download_cid_set.add(choice['cid'])
-                        node_infos.update({len(download_cid_set): (choice['cid'], choice['option'])})
-                        self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
-                        self.extract(**kwargs)
-                        self.download(**kwargs)
-                while len(search_node_list)>0:
-                    node_id = search_node_list.pop(0)
-                    params.update({'node_id':node_id})
-                    node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
-                    if node_info['data'].__contains__('edges'):
-                        for choice in node_info['data']['edges']['choices']:
-                            search_node_list.append(choice['node_id'])
-                            if not choice['cid'] in download_cid_set:
-                                download_cid_set.add(choice['cid'] )
-                                node_infos.update({len(download_cid_set):(choice['cid'],choice['option'])})
-                                self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
-                                self.extract(**kwargs)
-                                self.download(**kwargs)
-            else:
-                for pi in range(1, pn + 1):
-                    purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
-                    self.__class__().download_by_url(purl, **kwargs)
-
-        elif sort == 'bangumi':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            epn, i = len(initial_state['epList']), 0
-            for ep in initial_state['epList']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                ep_id = ep['id']
-                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
-                self.__class__().download_by_url(epurl, **kwargs)
-
-        elif sort == 'bangumi_md':
-            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
-            initial_state = json.loads(initial_state_text)
-            epn, i = len(initial_state['mediaInfo']['episodes']), 0
-            for ep in initial_state['mediaInfo']['episodes']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                ep_id = ep['ep_id']
-                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
-                self.__class__().download_by_url(epurl, **kwargs)
-
-        elif sort == 'space_channel':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
-            mid, cid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_channel_api(mid, cid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            channel_info = json.loads(api_content)
-            # TBD: channel of more than 100 videos
-
-            epn, i = len(channel_info['data']['list']['archives']), 0
-            for video in channel_info['data']['list']['archives']:
-                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'space_favlist':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
-            vmid, fid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_favlist_api(vmid, fid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            favlist_info = json.loads(api_content)
-            pc = favlist_info['data']['pagecount']
-
-            for pn in range(1, pc + 1):
-                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
-                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-                favlist_info = json.loads(api_content)
-
-                epn, i = len(favlist_info['data']['archives']), 0
-                for video in favlist_info['data']['archives']:
-                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                    self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'space_video':
-            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
-            mid = m.group(1)
-            api_url = self.bilibili_space_video_api(mid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            videos_info = json.loads(api_content)
-            pc = videos_info['data']['pages']
-
-            for pn in range(1, pc + 1):
-                api_url = self.bilibili_space_video_api(mid, pn=pn)
-                api_content = get_content(api_url, headers=self.bilibili_headers())
-                videos_info = json.loads(api_content)
-
-                epn, i = len(videos_info['data']['vlist']), 0
-                for video in videos_info['data']['vlist']:
-                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
-                    self.__class__().download_playlist_by_url(url, **kwargs)
-
-        elif sort == 'audio_menu':
-            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
-            sid = m.group(1)
-            #api_url = self.bilibili_audio_menu_info_api(sid)
-            #api_content = get_content(api_url, headers=self.bilibili_headers())
-            #menu_info = json.loads(api_content)
-            api_url = self.bilibili_audio_menu_song_api(sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
-            menusong_info = json.loads(api_content)
-            epn, i = len(menusong_info['data']['data']), 0
-            for song in menusong_info['data']['data']:
-                i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
-                url = 'https://www.bilibili.com/audio/au%s' % song['id']
-                self.__class__().download_by_url(url, **kwargs)
-
-
-site = Bilibili()
-download = site.download_by_url
-download_playlist = site.download_playlist_by_url
-
-bilibili_download = download
+#!/usr/bin/env python
+
+from ..common import *
+from ..extractor import VideoExtractor
+
+import hashlib
+
+class Bilibili(VideoExtractor):
+    name = "Bilibili"
+
+    # Bilibili media encoding options, in descending quality order.
+    stream_types = [
+        {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
+        {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P+'},
+        {'id': 'flv', 'quality': 80, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P'},
+        {'id': 'flv720_p60', 'quality': 74, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P60'},
+        {'id': 'flv720', 'quality': 64, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '720p', 'desc': '高清 720P'},
+        {'id': 'hdmp4', 'quality': 48, 'audio_quality': 30280,
+         'container': 'MP4', 'video_resolution': '720p', 'desc': '高清 720P (MP4)'},
+        {'id': 'flv480', 'quality': 32, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '480p', 'desc': '清晰 480P'},
+        {'id': 'flv360', 'quality': 16, 'audio_quality': 30216,
+         'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
+        # 'quality': 15?
+        {'id': 'mp4', 'quality': 0},
+    ]
+
+    @staticmethod
+    def height_to_quality(height):
+        if height <= 360:
+            return 16
+        elif height <= 480:
+            return 32
+        elif height <= 720:
+            return 64
+        else:
+            return 80
+
+    @staticmethod
+    def bilibili_headers(referer=None, cookie=None):
+        # a reasonable UA
+        ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
+        headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
+        if referer is not None:
+            headers.update({'Referer': referer})
+        if cookie is not None:
+            headers.update({'Cookie': cookie})
+        return headers
+
+    @staticmethod
+    def bilibili_api(avid, cid, qn=0):
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
+
+    @staticmethod
+    def bilibili_audio_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/url?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/info?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_menu_info_api(sid):
+        return 'https://www.bilibili.com/audio/music-service-c/web/menu/info?sid=%s' % sid
+
+    @staticmethod
+    def bilibili_audio_menu_song_api(sid, ps=100):
+        return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
+
+    @staticmethod
+    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
+        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
+
+    @staticmethod
+    def bilibili_interface_api(cid, qn=0):
+        entropy = 'rbMCKn@KuamXWlPMoJGsKcbiJKUfkPF_8dABscJntvqhRSETg'
+        appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
+        params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
+        chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
+        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+
+    @staticmethod
+    def bilibili_live_api(cid):
+        return 'https://api.live.bilibili.com/room/v1/Room/playUrl?cid=%s&quality=0&platform=web' % cid
+
+    @staticmethod
+    def bilibili_live_room_info_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/get_info?room_id=%s' % room_id
+
+    @staticmethod
+    def bilibili_live_room_init_api(room_id):
+        return 'https://api.live.bilibili.com/room/v1/Room/room_init?id=%s' % room_id
+
+    @staticmethod
+    def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
+
+    @staticmethod
+    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
+
+    @staticmethod
+    def bilibili_space_video_api(mid, pn=1, ps=100):
+        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
+
+    @staticmethod
+    def bilibili_vc_api(video_id):
+        return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
+
+    @staticmethod
+    def url_size(url, faker=False, headers={},err_value=0):
+        try:
+            return url_size(url,faker,headers)
+        except:
+            return err_value
+
+    def prepare(self, **kwargs):
+        self.stream_qualities = {s['quality']: s for s in self.stream_types}
+
+        try:
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+        except:
+            html_content = ''  # live always returns 400 (why?)
+        #self.title = match1(html_content,
+        #                    r'<h1 title="([^"]+)"')
+
+        # redirect: watchlater
+        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
+            avid = match1(self.url, r'/av(\d+)')
+            p = int(match1(self.url, r'/p(\d+)') or '1')
+            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # redirect: bangumi/play/ss -> bangumi/play/ep
+        # redirect: bangumi.bilibili.com/anime -> bangumi/play/ep
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ss(\d+)', self.url) or \
+             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)/play', self.url):
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            ep_id = initial_state['epList'][0]['id']
+            self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # sort it out
+        if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
+            sort = 'audio'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://live\.bilibili\.com/', self.url):
+            sort = 'live'
+        elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
+            sort = 'vc'
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+        else:
+            self.download_playlist_by_url(self.url, **kwargs)
+            return
+
+        # regular av video
+        if sort == 'video':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+
+            playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+            html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+            playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+            playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+
+            # warn if it is a multi-part video
+            pn = initial_state['videoData']['videos']
+            if pn > 1 and not kwargs.get('playlist'):
+                log.w('This is a multipart video. (use --playlist to download all parts.)')
+
+            # set video title
+            self.title = initial_state['videoData']['title']
+            # refine title for a specific part, if it is a multi-part video
+            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
+                    '1')  # use URL to decide p-number, not initial_state['p']
+            if pn > 1:
+                part = initial_state['videoData']['pages'][p - 1]['part']
+                self.title = '%s (P%s. %s)' % (self.title, p, part)
+
+            # construct playinfos
+            avid = initial_state['aid']
+            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
+            current_quality, best_quality = None, None
+            if playinfo is not None:
+                current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
+                if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
+                    best_quality = playinfo['data']['accept_quality'][0]
+            playinfos = []
+            if playinfo is not None:
+                playinfos.append(playinfo)
+            if playinfo_ is not None:
+                playinfos.append(playinfo_)
+            # get alternative formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # for dash, qn does not matter
+                if current_quality is None or qn < current_quality:
+                    api_url = self.bilibili_api(avid, cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
+                    else:
+                        message = api_playinfo['data']['message']
+                if best_quality is None or qn <= best_quality:
+                    api_url = self.bilibili_interface_api(cid, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo_data = json.loads(api_content)
+                    if api_playinfo_data.get('quality'):
+                        playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
+            if not playinfos:
+                log.w(message)
+                # use bilibili error video instead
+                url = 'https://static.hdslb.com/error.mp4'
+                _, container, size = url_info(url)
+                self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
+                return
+
+            for playinfo in playinfos:
+                quality = playinfo['data']['quality']
+                format_id = self.stream_qualities[quality]['id']
+                container = self.stream_qualities[quality]['container'].lower()
+                desc = self.stream_qualities[quality]['desc']
+
+                if 'durl' in playinfo['data']:
+                    src, size = [], 0
+                    for durl in playinfo['data']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['data']:
+                    audio_size_cache = {}
+                    for video in playinfo['data']['dash']['video']:
+                        # prefer the latter codecs!
+                        s = self.stream_qualities[video['id']]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['data']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        if not audio_size_cache.get(audio_quality, False):
+                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                        size += audio_size_cache[audio_quality]
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+        # bangumi
+        elif sort == 'bangumi':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+
+            # warn if this bangumi has more than 1 video
+            epn = len(initial_state['epList'])
+            if epn > 1 and not kwargs.get('playlist'):
+                log.w('This bangumi currently has %s videos. (use --playlist to download all videos.)' % epn)
+
+            # set video title
+            self.title = initial_state['h1Title']
+
+            # construct playinfos
+            ep_id = initial_state['epInfo']['id']
+            avid = initial_state['epInfo']['aid']
+            cid = initial_state['epInfo']['cid']
+            playinfos = []
+            api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_playinfo = json.loads(api_content)
+            if api_playinfo['code'] == 0:  # success
+                playinfos.append(api_playinfo)
+            else:
+                log.e(api_playinfo['message'])
+                return
+            current_quality = api_playinfo['result']['quality']
+            # get alternative formats from API
+            for qn in [80, 64, 32, 16]:
+                # automatic format for durl: qn=0
+                # for dash, qn does not matter
+                if qn != current_quality:
+                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
+                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_playinfo = json.loads(api_content)
+                    if api_playinfo['code'] == 0:  # success
+                        playinfos.append(api_playinfo)
+
+            for playinfo in playinfos:
+                if 'durl' in playinfo['result']:
+                    quality = playinfo['result']['quality']
+                    format_id = self.stream_qualities[quality]['id']
+                    container = self.stream_qualities[quality]['container'].lower()
+                    desc = self.stream_qualities[quality]['desc']
+
+                    src, size = [], 0
+                    for durl in playinfo['result']['durl']:
+                        src.append(durl['url'])
+                        size += durl['size']
+                    self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+                # DASH formats
+                if 'dash' in playinfo['result']:
+                    for video in playinfo['result']['dash']['video']:
+                        # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
+                        quality = self.height_to_quality(video['height'])  # convert height to quality code
+                        s = self.stream_qualities[quality]
+                        format_id = 'dash-' + s['id']  # prefix
+                        container = 'mp4'  # enforce MP4 container
+                        desc = s['desc']
+                        audio_quality = s['audio_quality']
+                        baseurl = video['baseUrl']
+                        size = url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        # find matching audio track
+                        audio_baseurl = playinfo['result']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['result']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        size += url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+            # get danmaku
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+        # vc video
+        elif sort == 'vc':
+            video_id = match1(self.url, r'https?://vc\.?bilibili\.com/video/(\d+)')
+            api_url = self.bilibili_vc_api(video_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_playinfo = json.loads(api_content)
+
+            # set video title
+            self.title = '%s (%s)' % (api_playinfo['data']['user']['name'], api_playinfo['data']['item']['id'])
+
+            height = api_playinfo['data']['item']['height']
+            quality = self.height_to_quality(height)  # convert height to quality code
+            s = self.stream_qualities[quality]
+            format_id = s['id']
+            container = 'mp4'  # enforce MP4 container
+            desc = s['desc']
+
+            playurl = api_playinfo['data']['item']['video_playurl']
+            size = int(api_playinfo['data']['item']['video_size'])
+
+            self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': [playurl]}
+
+        # live
+        elif sort == 'live':
+            m = re.match(r'https?://live\.bilibili\.com/(\w+)', self.url)
+            short_id = m.group(1)
+            api_url = self.bilibili_live_room_init_api(short_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_init_info = json.loads(api_content)
+
+            room_id = room_init_info['data']['room_id']
+            api_url = self.bilibili_live_room_info_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            room_info = json.loads(api_content)
+
+            # set video title
+            self.title = room_info['data']['title'] + '.' + str(int(time.time()))
+
+            api_url = self.bilibili_live_api(room_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            video_info = json.loads(api_content)
+
+            durls = video_info['data']['durl']
+            playurl = durls[0]['url']
+            container = 'flv'  # enforce FLV container
+            self.streams['flv'] = {'container': container, 'quality': 'unknown',
+                                   'size': 0, 'src': [playurl]}
+
+        # audio
+        elif sort == 'audio':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/au(\d+)', self.url)
+            sid = m.group(1)
+            api_url = self.bilibili_audio_info_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            song_info = json.loads(api_content)
+
+            # set audio title
+            self.title = song_info['data']['title']
+
+            # get lyrics
+            self.lyrics = get_content(song_info['data']['lyric'])
+
+            api_url = self.bilibili_audio_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            audio_info = json.loads(api_content)
+
+            playurl = audio_info['data']['cdns'][0]
+            size = audio_info['data']['size']
+            container = 'mp4'  # enforce MP4 container
+            self.streams['mp4'] = {'container': container,
+                                   'size': size, 'src': [playurl]}
+
+
+    def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
+        #response for interaction video
+        #主要针对互动视频，使用cid而不是url来相互区分
+
+        self.stream_qualities = {s['quality']: s for s in self.stream_types}
+        self.title = title
+        self.url = url
+
+        current_quality, best_quality = None, None
+        if playinfo is not None:
+            current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None
+            if 'accept_quality' in playinfo['data'] and playinfo['data']['accept_quality'] != []:
+                best_quality = playinfo['data']['accept_quality'][0]
+        playinfos = []
+        if playinfo is not None:
+            playinfos.append(playinfo)
+        if playinfo_ is not None:
+            playinfos.append(playinfo_)
+        # get alternative formats from API
+        for qn in [80, 64, 32, 16]:
+            # automatic format for durl: qn=0
+            # for dash, qn does not matter
+            if current_quality is None or qn < current_quality:
+                api_url = self.bilibili_api(avid, cid, qn=qn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                api_playinfo = json.loads(api_content)
+                if api_playinfo['code'] == 0:  # success
+                    playinfos.append(api_playinfo)
+                else:
+                    message = api_playinfo['data']['message']
+            if best_quality is None or qn <= best_quality:
+                api_url = self.bilibili_interface_api(cid, qn=qn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                api_playinfo_data = json.loads(api_content)
+                if api_playinfo_data.get('quality'):
+                    playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
+        if not playinfos:
+            log.w(message)
+            # use bilibili error video instead
+            url = 'https://static.hdslb.com/error.mp4'
+            _, container, size = url_info(url)
+            self.streams['flv480'] = {'container': container, 'size': size, 'src': [url]}
+            return
+
+        for playinfo in playinfos:
+            quality = playinfo['data']['quality']
+            format_id = self.stream_qualities[quality]['id']
+            container = self.stream_qualities[quality]['container'].lower()
+            desc = self.stream_qualities[quality]['desc']
+
+            if 'durl' in playinfo['data']:
+                src, size = [], 0
+                for durl in playinfo['data']['durl']:
+                    src.append(durl['url'])
+                    size += durl['size']
+                self.streams[format_id] = {'container': container, 'quality': desc, 'size': size, 'src': src}
+
+            # DASH formats
+            if 'dash' in playinfo['data']:
+                audio_size_cache = {}
+                for video in playinfo['data']['dash']['video']:
+                    # prefer the latter codecs!
+                    s = self.stream_qualities[video['id']]
+                    format_id = 'dash-' + s['id']  # prefix
+                    container = 'mp4'  # enforce MP4 container
+                    desc = s['desc']
+                    audio_quality = s['audio_quality']
+                    baseurl = video['baseUrl']
+                    size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
+
+                    # find matching audio track
+                    audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                    for audio in playinfo['data']['dash']['audio']:
+                        if int(audio['id']) == audio_quality:
+                            audio_baseurl = audio['baseUrl']
+                            break
+                    if not audio_size_cache.get(audio_quality, False):
+                        audio_size_cache[audio_quality] = self.url_size(audio_baseurl,
+                                                                        headers=self.bilibili_headers(referer=self.url))
+                    size += audio_size_cache[audio_quality]
+
+                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
+
+        # get danmaku
+        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+
+    def extract(self, **kwargs):
+        # set UA and referer for downloading
+        headers = self.bilibili_headers(referer=self.url)
+        self.ua, self.referer = headers['User-Agent'], headers['Referer']
+
+        if not self.streams_sorted:
+            # no stream is available
+            return
+
+        if 'stream_id' in kwargs and kwargs['stream_id']:
+            # extract the stream
+            stream_id = kwargs['stream_id']
+            if stream_id not in self.streams and stream_id not in self.dash_streams:
+                log.e('[Error] Invalid video format.')
+                log.e('Run \'-i\' command with no specific video format to view all available formats.')
+                exit(2)
+        else:
+            # extract stream with the best quality
+            stream_id = self.streams_sorted[0]['id']
+
+    def download_playlist_by_url(self, url, **kwargs):
+        self.url = url
+        kwargs['playlist'] = True
+
+        html_content = get_content(self.url, headers=self.bilibili_headers())
+
+        # sort it out
+        if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):
+            sort = 'bangumi'
+        elif match1(html_content, r'<meta property="og:url" content="(https://www.bilibili.com/bangumi/play/[^"]+)"'):
+            sort = 'bangumi'
+        elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
+            re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
+            sort = 'bangumi_md'
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+            sort = 'video'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
+            sort = 'space_channel'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
+            sort = 'space_favlist'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
+            sort = 'space_video'
+        elif re.match(r'https?://(www\.)?bilibili\.com/audio/am(\d+)', self.url):
+            sort = 'audio_menu'
+        else:
+            log.e('[Error] Unsupported URL pattern.')
+            exit(1)
+
+        # regular av video
+        if sort == 'video':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            aid = initial_state['videoData']['aid']
+            pn = initial_state['videoData']['videos']
+            if pn!= len(initial_state['videoData']['pages']):#interaction video 互动视频
+                search_node_list = []
+                download_cid_set = set([initial_state['videoData']['cid']])
+                params = {
+                        'id': 'cid:{}'.format(initial_state['videoData']['cid']),
+                        'aid': str(aid)
+                }
+                urlcontent = get_content('https://api.bilibili.com/x/player.so?'+parse.urlencode(params), headers=self.bilibili_headers(referer='https://www.bilibili.com/video/av{}'.format(aid)))
+                graph_version = json.loads(urlcontent[urlcontent.find('<interaction>')+13:urlcontent.find('</interaction>')])['graph_version']
+                params = {
+                    'aid': str(aid),
+                    'graph_version': graph_version,
+                    'platform': 'pc',
+                    'portal': 0,
+                    'screen': 0,
+                }
+                node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
+
+                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+
+                self.prepare_by_cid(aid, initial_state['videoData']['cid'], initial_state['videoData']['title'] + ('P{}. {}'.format(1, node_info['data']['title'])),html_content,playinfo,playinfo_,url)
+                self.extract(**kwargs)
+                self.download(**kwargs)
+                for choice in node_info['data']['edges']['choices']:
+                    search_node_list.append(choice['node_id'])
+                    if not choice['cid'] in download_cid_set:
+                        download_cid_set.add(choice['cid'])
+                        self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
+                        self.extract(**kwargs)
+                        self.download(**kwargs)
+                while len(search_node_list)>0:
+                    node_id = search_node_list.pop(0)
+                    params.update({'node_id':node_id})
+                    node_info = json.loads(get_content('https://api.bilibili.com/x/stein/nodeinfo?'+parse.urlencode(params)))
+                    if node_info['data'].__contains__('edges'):
+                        for choice in node_info['data']['edges']['choices']:
+                            search_node_list.append(choice['node_id'])
+                            if not choice['cid'] in download_cid_set:
+                                download_cid_set.add(choice['cid'] )
+                                self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
+                                self.extract(**kwargs)
+                                self.download(**kwargs)
+            else:
+                for pi in range(1, pn + 1):
+                    purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
+                    self.__class__().download_by_url(purl, **kwargs)
+
+        elif sort == 'bangumi':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['epList']), 0
+            for ep in initial_state['epList']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                ep_id = ep['id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+
+        elif sort == 'bangumi_md':
+            initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
+            initial_state = json.loads(initial_state_text)
+            epn, i = len(initial_state['mediaInfo']['episodes']), 0
+            for ep in initial_state['mediaInfo']['episodes']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                ep_id = ep['ep_id']
+                epurl = 'https://www.bilibili.com/bangumi/play/ep%s/' % ep_id
+                self.__class__().download_by_url(epurl, **kwargs)
+
+        elif sort == 'space_channel':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url)
+            mid, cid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_channel_api(mid, cid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            channel_info = json.loads(api_content)
+            # TBD: channel of more than 100 videos
+
+            epn, i = len(channel_info['data']['list']['archives']), 0
+            for video in channel_info['data']['list']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'space_favlist':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
+            vmid, fid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_favlist_api(vmid, fid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            favlist_info = json.loads(api_content)
+            pc = favlist_info['data']['pagecount']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                favlist_info = json.loads(api_content)
+
+                epn, i = len(favlist_info['data']['archives']), 0
+                for video in favlist_info['data']['archives']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'space_video':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url)
+            mid = m.group(1)
+            api_url = self.bilibili_space_video_api(mid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            videos_info = json.loads(api_content)
+            pc = videos_info['data']['pages']
+
+            for pn in range(1, pc + 1):
+                api_url = self.bilibili_space_video_api(mid, pn=pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers())
+                videos_info = json.loads(api_content)
+
+                epn, i = len(videos_info['data']['vlist']), 0
+                for video in videos_info['data']['vlist']:
+                    i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    self.__class__().download_playlist_by_url(url, **kwargs)
+
+        elif sort == 'audio_menu':
+            m = re.match(r'https?://(?:www\.)?bilibili\.com/audio/am(\d+)', self.url)
+            sid = m.group(1)
+            #api_url = self.bilibili_audio_menu_info_api(sid)
+            #api_content = get_content(api_url, headers=self.bilibili_headers())
+            #menu_info = json.loads(api_content)
+            api_url = self.bilibili_audio_menu_song_api(sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            menusong_info = json.loads(api_content)
+            epn, i = len(menusong_info['data']['data']), 0
+            for song in menusong_info['data']['data']:
+                i += 1; log.w('Extracting %s of %s songs ...' % (i, epn))
+                url = 'https://www.bilibili.com/audio/au%s' % song['id']
+                self.__class__().download_by_url(url, **kwargs)
+
+
+site = Bilibili()
+download = site.download_by_url
+download_playlist = site.download_playlist_by_url
+
+bilibili_download = download

From 73bdc2d0012a261e3d63a997277f19a515057b4f Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Thu, 3 Oct 2019 14:43:32 +0800
Subject: [PATCH 0893/1225] Add simple support for missevan.com * single audio
 file download * album download by --playlist option

---
 src/you_get/common.py              |   1 +
 src/you_get/extractors/missevan.py | 223 +++++++++++++++++++++++++++++
 2 files changed, 224 insertions(+)
 create mode 100644 src/you_get/extractors/missevan.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9409462da3..5b16fc0490 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -79,6 +79,7 @@
     'metacafe'         : 'metacafe',
     'mgtv'             : 'mgtv',
     'miomio'           : 'miomio',
+    'missevan'         : 'missevan',
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
     'musicplayon'      : 'musicplayon',
diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
new file mode 100644
index 0000000000..5c4195bad7
--- /dev/null
+++ b/src/you_get/extractors/missevan.py
@@ -0,0 +1,223 @@
+"""
+MIT License
+
+Copyright (c) 2019 WaferJay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import json
+import re
+
+from ..common import get_content, urls_size, log
+from ..extractor import VideoExtractor
+
+
+class NoMatchException(Exception):
+    pass
+
+
+class _Dispatcher(object):
+
+    def __init__(self):
+        self.entry = []
+
+    def register(self, patterns, fun):
+        if not isinstance(patterns, (list, tuple)):
+            patterns = [patterns]
+
+        patterns = [re.compile(reg) for reg in patterns]
+        self.entry.append((patterns, fun))
+
+    def endpoint(self, *patterns):
+        def _wrap(fun):
+            self.register(patterns, fun)
+            return fun
+        return _wrap
+
+    def dispatch(self, url, *args, **kwargs):
+
+        for patterns, fun in self.entry:
+
+            for pa in patterns:
+
+                match = pa.search(url)
+                if not match:
+                    continue
+
+                kwargs.update(match.groupdict())
+                return fun(*args, **kwargs)
+
+        raise NoMatchException()
+
+
+missevan_stream_types = [
+    {'id': '128bit', 'url_json_key': 'soundurl_32'},
+    {'id': '64bit', 'url_json_key': 'soundurl_64'},
+    {'id': '32bit', 'url_json_key': 'soundurl_128'}
+]
+
+
+class MissEvanWithStream(VideoExtractor):
+
+    name = 'MissEvan'
+    stream_types = missevan_stream_types
+
+    @classmethod
+    def create(cls, title, streams, streams_sorted=None):
+        obj = cls()
+        obj.title = title
+        obj.streams.update(streams)
+        streams_sorted = streams_sorted or cls._setup_streams_sorted(streams)
+        obj.streams_sorted.extend(streams_sorted)
+        return obj
+
+    @staticmethod
+    def _setup_streams_sorted(streams):
+        streams_sorted = []
+        for key, stream in streams.items():
+            copy_stream = stream.copy()
+            copy_stream['id'] = key
+            streams_sorted.append(copy_stream)
+
+        return streams_sorted
+
+    def download(self, **kwargs):
+        stream_id = kwargs.get('stream_id') or self.stream_types[0]['id']
+        stream = self.streams[stream_id]
+        if 'size' not in stream:
+            stream['size'] = urls_size(stream['src'])
+
+        super().download(**kwargs)
+
+    def unsupported_method(self, *args, **kwargs):
+        raise AssertionError('Unsupported')
+
+    download_by_url = unsupported_method
+    download_by_vid = unsupported_method
+    prepare = unsupported_method
+    extract = unsupported_method
+
+
+class MissEvan(VideoExtractor):
+
+    name = 'MissEvan'
+    stream_types = missevan_stream_types
+
+    _SOUND_URL_PREFIX = 'https://static.missevan.com/'
+
+    _P_ALBUM_URL = re.compile(r'missevan\.com/album(?:info)?/(?P<aid>\d+)', re.I)
+
+    @classmethod
+    def is_album_url(cls, url):
+        return bool(cls._P_ALBUM_URL.search(url))
+
+    @classmethod
+    def parse_album_id(cls, url):
+        match = cls._P_ALBUM_URL.search(url)
+        if not match:
+            raise ValueError()
+        return match.group('aid')
+
+    def download_by_url(self, url, **kwargs):
+        if not kwargs.get('playlist') and self.is_album_url(url):
+            log.w('This is an album. (use --playlist to download all audios).')
+        else:
+            super().download_by_url(url, **kwargs)
+
+    __prepare_dispatcher = _Dispatcher()
+
+    @__prepare_dispatcher.endpoint(
+        re.compile(r'missevan\.com/sound/(?:player\?.*?id=)?(?P<sid>\d+)', re.I))
+    def prepare_sound(self, sid, **kwargs):
+        content = get_content('https://www.missevan.com/sound/getsound?soundid=' + sid)
+        json_data = json.loads(content)
+        sound = json_data['info']['sound']
+
+        q32_url = self._SOUND_URL_PREFIX + sound['soundurl_32']
+        q64_url = self._SOUND_URL_PREFIX + sound['soundurl_64']
+        q128_url = self._SOUND_URL_PREFIX + sound['soundurl_128']
+
+        self.streams = {
+            '32bit': {
+                'src': [q32_url],
+                'container': 'mp3'
+            },
+            '64bit': {
+                'src': [q64_url],
+                'container': 'mp3'
+            },
+            '128bit': {
+                'src': [q128_url],
+                'container': 'mp3'
+            }
+        }
+        self.title = sound['soundstr']
+
+    def prepare(self, **kwargs):
+        if self.vid:
+            self.prepare_sound(self.vid, **kwargs)
+            return
+
+        try:
+            self.__prepare_dispatcher.dispatch(self.url, self, **kwargs)
+        except NoMatchException:
+            log.e('[Error] Unsupported URL pattern.')
+            exit(1)
+
+    def download_playlist_by_url(self, url, **kwargs):
+        self.url = url
+        aid = None
+        try:
+            aid = self.parse_album_id(url)
+        except ValueError:
+            log.e('[Error] Unsupported URL pattern with --playlist option.')
+            exit(1)
+
+        # use the best quality by default
+        kwargs.setdefault('stream_id', missevan_stream_types[0]['id'])
+
+        content = get_content('https://www.missevan.com/sound/soundalllist?albumid=' + aid)
+        json_data = json.loads(content)
+        album = json_data['info']['album']
+        self.title = album['title']
+        sounds = json_data['info']['sounds']
+
+        for sound in sounds:
+            streams = {}
+
+            for stream_type in missevan_stream_types:
+                sound_url = self._SOUND_URL_PREFIX + sound[stream_type['url_json_key']]
+                streams[stream_type['id']] = {'src': [sound_url], 'container': 'mp3'}
+
+            sound_title = sound['soundstr']
+            MissEvanWithStream \
+                .create(sound_title, streams) \
+                .download(**kwargs)
+
+    def extract(self, **kwargs):
+        stream_id = kwargs.get('stream_id') or missevan_stream_types[0]['id']
+        stream = self.streams[stream_id]
+        if 'size' not in stream:
+            stream['size'] = urls_size(stream['src'])
+
+site = MissEvan()
+site_info = 'MissEvan.com'
+download = site.download_by_url
+download_playlist = site.download_playlist_by_url

From 5fc5fa8fcd6824b1842d692cc8eda77f44829174 Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Thu, 3 Oct 2019 15:24:37 +0800
Subject: [PATCH 0894/1225] Optimize code

---
 src/you_get/extractors/missevan.py | 47 +++++++++++++-----------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 5c4195bad7..6b9cdf421c 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -120,8 +120,6 @@ class MissEvan(VideoExtractor):
     name = 'MissEvan'
     stream_types = missevan_stream_types
 
-    _SOUND_URL_PREFIX = 'https://static.missevan.com/'
-
     _P_ALBUM_URL = re.compile(r'missevan\.com/album(?:info)?/(?P<aid>\d+)', re.I)
 
     @classmethod
@@ -146,29 +144,14 @@ def download_by_url(self, url, **kwargs):
     @__prepare_dispatcher.endpoint(
         re.compile(r'missevan\.com/sound/(?:player\?.*?id=)?(?P<sid>\d+)', re.I))
     def prepare_sound(self, sid, **kwargs):
-        content = get_content('https://www.missevan.com/sound/getsound?soundid=' + sid)
+        content = get_content(self.url_sound_api(sid))
         json_data = json.loads(content)
         sound = json_data['info']['sound']
 
-        q32_url = self._SOUND_URL_PREFIX + sound['soundurl_32']
-        q64_url = self._SOUND_URL_PREFIX + sound['soundurl_64']
-        q128_url = self._SOUND_URL_PREFIX + sound['soundurl_128']
-
-        self.streams = {
-            '32bit': {
-                'src': [q32_url],
-                'container': 'mp3'
-            },
-            '64bit': {
-                'src': [q64_url],
-                'container': 'mp3'
-            },
-            '128bit': {
-                'src': [q128_url],
-                'container': 'mp3'
-            }
-        }
         self.title = sound['soundstr']
+        for stream_type in self.stream_types:
+            sound_url = self.url_resource(sound[stream_type['url_json_key']])
+            self.streams[stream_type['id']] = {'src': [sound_url], 'container': 'mp3'}
 
     def prepare(self, **kwargs):
         if self.vid:
@@ -191,9 +174,9 @@ def download_playlist_by_url(self, url, **kwargs):
             exit(1)
 
         # use the best quality by default
-        kwargs.setdefault('stream_id', missevan_stream_types[0]['id'])
+        kwargs.setdefault('stream_id', self.stream_types[0]['id'])
 
-        content = get_content('https://www.missevan.com/sound/soundalllist?albumid=' + aid)
+        content = get_content(self.url_album_api(aid))
         json_data = json.loads(content)
         album = json_data['info']['album']
         self.title = album['title']
@@ -202,8 +185,8 @@ def download_playlist_by_url(self, url, **kwargs):
         for sound in sounds:
             streams = {}
 
-            for stream_type in missevan_stream_types:
-                sound_url = self._SOUND_URL_PREFIX + sound[stream_type['url_json_key']]
+            for stream_type in self.stream_types:
+                sound_url = self.url_resource(sound[stream_type['url_json_key']])
                 streams[stream_type['id']] = {'src': [sound_url], 'container': 'mp3'}
 
             sound_title = sound['soundstr']
@@ -212,11 +195,23 @@ def download_playlist_by_url(self, url, **kwargs):
                 .download(**kwargs)
 
     def extract(self, **kwargs):
-        stream_id = kwargs.get('stream_id') or missevan_stream_types[0]['id']
+        stream_id = kwargs.get('stream_id') or self.stream_types[0]['id']
         stream = self.streams[stream_id]
         if 'size' not in stream:
             stream['size'] = urls_size(stream['src'])
 
+    @staticmethod
+    def url_album_api(album_id):
+        return 'https://www.missevan.com/sound/soundalllist?albumid=' + str(album_id)
+
+    @staticmethod
+    def url_sound_api(sound_id):
+        return 'https://www.missevan.com/sound/getsound?soundid=' + str(sound_id)
+
+    @staticmethod
+    def url_resource(uri):
+        return 'https://static.missevan.com/' + uri
+
 site = MissEvan()
 site_info = 'MissEvan.com'
 download = site.download_by_url

From 58a9823fe547b7337f6f8ac68a31c709fb5e844a Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Thu, 3 Oct 2019 16:46:55 +0800
Subject: [PATCH 0895/1225] [missevan] Add support for drama download

---
 src/you_get/extractors/missevan.py | 70 +++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 26 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 6b9cdf421c..885cc8f47a 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -23,6 +23,7 @@
 """
 
 import json
+import os
 import re
 
 from ..common import get_content, urls_size, log
@@ -46,11 +47,15 @@ def register(self, patterns, fun):
         self.entry.append((patterns, fun))
 
     def endpoint(self, *patterns):
+        assert patterns, 'patterns must not be empty'
         def _wrap(fun):
             self.register(patterns, fun)
             return fun
         return _wrap
 
+    def test(self, url):
+        return any(pa.search(url) for pas, _ in self.entry for pa in pas)
+
     def dispatch(self, url, *args, **kwargs):
 
         for patterns, fun in self.entry:
@@ -120,22 +125,9 @@ class MissEvan(VideoExtractor):
     name = 'MissEvan'
     stream_types = missevan_stream_types
 
-    _P_ALBUM_URL = re.compile(r'missevan\.com/album(?:info)?/(?P<aid>\d+)', re.I)
-
-    @classmethod
-    def is_album_url(cls, url):
-        return bool(cls._P_ALBUM_URL.search(url))
-
-    @classmethod
-    def parse_album_id(cls, url):
-        match = cls._P_ALBUM_URL.search(url)
-        if not match:
-            raise ValueError()
-        return match.group('aid')
-
     def download_by_url(self, url, **kwargs):
-        if not kwargs.get('playlist') and self.is_album_url(url):
-            log.w('This is an album. (use --playlist to download all audios).')
+        if not kwargs.get('playlist') and self._download_playlist_dispatcher.test(url):
+            log.w('This is an album or drama. (use --playlist option to download all).')
         else:
             super().download_by_url(url, **kwargs)
 
@@ -164,18 +156,11 @@ def prepare(self, **kwargs):
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
 
-    def download_playlist_by_url(self, url, **kwargs):
-        self.url = url
-        aid = None
-        try:
-            aid = self.parse_album_id(url)
-        except ValueError:
-            log.e('[Error] Unsupported URL pattern with --playlist option.')
-            exit(1)
-
-        # use the best quality by default
-        kwargs.setdefault('stream_id', self.stream_types[0]['id'])
+    _download_playlist_dispatcher = _Dispatcher()
 
+    @_download_playlist_dispatcher.endpoint(
+        re.compile(r'missevan\.com/album(?:info)?/(?P<aid>\d+)', re.I))
+    def download_album(self, aid, **kwargs):
         content = get_content(self.url_album_api(aid))
         json_data = json.loads(content)
         album = json_data['info']['album']
@@ -194,6 +179,35 @@ def download_playlist_by_url(self, url, **kwargs):
                 .create(sound_title, streams) \
                 .download(**kwargs)
 
+    @_download_playlist_dispatcher.endpoint(
+        re.compile(r'missevan\.com(?:/mdrama)?/drama/(?P<did>\d+)', re.I))
+    def download_drama(self, did, **kwargs):
+        content = get_content(self.url_drama_api(did))
+        json_data = json.loads(content)
+
+        drama = json_data['info']['drama']
+
+        self.title = drama['name']
+        output_dir = os.path.abspath(kwargs.pop('output_dir', '.'))
+        output_dir = os.path.join(output_dir, self.title)
+        kwargs['output_dir'] = output_dir
+
+        episodes = json_data['info']['episodes']
+        for each in episodes['episode']:
+            sound_id = each['sound_id']
+            MissEvan().download_by_vid(sound_id, **kwargs)
+
+    def download_playlist_by_url(self, url, **kwargs):
+        # use the best quality by default
+        kwargs.setdefault('stream_id', self.stream_types[0]['id'])
+
+        self.url = url
+        try:
+            self._download_playlist_dispatcher.dispatch(url, self, **kwargs)
+        except NoMatchException:
+            log.e('[Error] Unsupported URL pattern with --playlist option.')
+            exit(1)
+
     def extract(self, **kwargs):
         stream_id = kwargs.get('stream_id') or self.stream_types[0]['id']
         stream = self.streams[stream_id]
@@ -208,6 +222,10 @@ def url_album_api(album_id):
     def url_sound_api(sound_id):
         return 'https://www.missevan.com/sound/getsound?soundid=' + str(sound_id)
 
+    @staticmethod
+    def url_drama_api(drama_id):
+        return 'https://www.missevan.com/dramaapi/getdrama?drama_id=' + str(drama_id)
+
     @staticmethod
     def url_resource(uri):
         return 'https://static.missevan.com/' + uri

From 5ddd9fb1e56cc3f4c4a5e9f29e4ca1de55230301 Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Thu, 3 Oct 2019 20:18:35 +0800
Subject: [PATCH 0896/1225] [missevan] download sound with cover by default

---
 src/you_get/extractors/missevan.py | 100 ++++++++++++++++++++++-------
 1 file changed, 77 insertions(+), 23 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 885cc8f47a..c3cc82b9e1 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -30,7 +30,7 @@
 from ..extractor import VideoExtractor
 
 
-class NoMatchException(Exception):
+class _NoMatchException(Exception):
     pass
 
 
@@ -69,15 +69,28 @@ def dispatch(self, url, *args, **kwargs):
                 kwargs.update(match.groupdict())
                 return fun(*args, **kwargs)
 
-        raise NoMatchException()
+        raise _NoMatchException()
 
 
 missevan_stream_types = [
-    {'id': '128bit', 'url_json_key': 'soundurl_32'},
-    {'id': '64bit', 'url_json_key': 'soundurl_64'},
-    {'id': '32bit', 'url_json_key': 'soundurl_128'}
+    {'id': '128bit', 'url_json_key': 'soundurl_128', 'desc': '128 Kbps'},
+    {'id': '64bit', 'url_json_key': 'soundurl_64', 'desc': '64 Kbps'},
+    {'id': '32bit', 'url_json_key': 'soundurl_32', 'desc': '32 Kbps'},
+    {'id': 'covers', 'url_json_key': 'cover_image', 'desc': '封面图'},
+    {'id': 'coversmini', 'url_json_key': 'cover_image', 'desc': '封面缩略图'}
 ]
 
+def is_covers_stream(stream):
+    stream = stream or ''
+    return stream.lower() in ('covers', 'coversmini')
+
+def get_file_extension(file_path, default=''):
+    _, suffix = os.path.splitext(file_path)
+    if suffix:
+        # remove dot
+        suffix = suffix[1:]
+    return suffix or default
+
 
 class MissEvanWithStream(VideoExtractor):
 
@@ -85,7 +98,7 @@ class MissEvanWithStream(VideoExtractor):
     stream_types = missevan_stream_types
 
     @classmethod
-    def create(cls, title, streams, streams_sorted=None):
+    def create(cls, title, streams, *, streams_sorted=None):
         obj = cls()
         obj.title = title
         obj.streams.update(streams)
@@ -93,6 +106,10 @@ def create(cls, title, streams, streams_sorted=None):
         obj.streams_sorted.extend(streams_sorted)
         return obj
 
+    def fetch_danmaku(self, url, headers=None):
+        self.danmaku = get_content(url, headers or {})
+        return self
+
     @staticmethod
     def _setup_streams_sorted(streams):
         streams_sorted = []
@@ -125,12 +142,6 @@ class MissEvan(VideoExtractor):
     name = 'MissEvan'
     stream_types = missevan_stream_types
 
-    def download_by_url(self, url, **kwargs):
-        if not kwargs.get('playlist') and self._download_playlist_dispatcher.test(url):
-            log.w('This is an album or drama. (use --playlist option to download all).')
-        else:
-            super().download_by_url(url, **kwargs)
-
     __prepare_dispatcher = _Dispatcher()
 
     @__prepare_dispatcher.endpoint(
@@ -141,9 +152,26 @@ def prepare_sound(self, sid, **kwargs):
         sound = json_data['info']['sound']
 
         self.title = sound['soundstr']
-        for stream_type in self.stream_types:
-            sound_url = self.url_resource(sound[stream_type['url_json_key']])
-            self.streams[stream_type['id']] = {'src': [sound_url], 'container': 'mp3'}
+        if not is_covers_stream(kwargs.get('stream_id')):
+            self.danmaku = get_content(self.url_danmaku_api(sid))
+
+        self.streams = self.setup_streams(sound)
+
+    @classmethod
+    def setup_streams(cls, sound):
+        streams = {}
+
+        for stream_type in cls.stream_types:
+            stream_id = stream_type['id']
+            uri = sound[stream_type['url_json_key']]
+            if is_covers_stream(stream_id):
+                resource_url = cls.url_resource(stream_id + '/' + uri)
+            else:
+                resource_url = cls.url_resource(uri)
+
+            container = get_file_extension(uri)
+            streams[stream_id] = {'src': [resource_url], 'container': container}
+        return streams
 
     def prepare(self, **kwargs):
         if self.vid:
@@ -152,10 +180,18 @@ def prepare(self, **kwargs):
 
         try:
             self.__prepare_dispatcher.dispatch(self.url, self, **kwargs)
-        except NoMatchException:
+        except _NoMatchException:
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
 
+    @staticmethod
+    def download_covers(title, streams, **kwargs):
+        if not is_covers_stream(kwargs.get('stream_id')):
+            kwargs['stream_id'] = 'covers'
+            MissEvanWithStream \
+                .create(title, streams) \
+                .download(**kwargs)
+
     _download_playlist_dispatcher = _Dispatcher()
 
     @_download_playlist_dispatcher.endpoint(
@@ -167,18 +203,21 @@ def download_album(self, aid, **kwargs):
         self.title = album['title']
         sounds = json_data['info']['sounds']
 
-        for sound in sounds:
-            streams = {}
-
-            for stream_type in self.stream_types:
-                sound_url = self.url_resource(sound[stream_type['url_json_key']])
-                streams[stream_type['id']] = {'src': [sound_url], 'container': 'mp3'}
+        output_dir = os.path.abspath(kwargs.pop('output_dir', '.'))
+        output_dir = os.path.join(output_dir, self.title)
+        kwargs['output_dir'] = output_dir
 
+        for sound in sounds:
+            streams = self.setup_streams(sound)
+            sound_id = sound['id']
             sound_title = sound['soundstr']
             MissEvanWithStream \
                 .create(sound_title, streams) \
+                .fetch_danmaku(self.url_danmaku_api(sound_id)) \
                 .download(**kwargs)
 
+            self.download_covers(sound_title, streams, **kwargs)
+
     @_download_playlist_dispatcher.endpoint(
         re.compile(r'missevan\.com(?:/mdrama)?/drama/(?P<did>\d+)', re.I))
     def download_drama(self, did, **kwargs):
@@ -204,10 +243,21 @@ def download_playlist_by_url(self, url, **kwargs):
         self.url = url
         try:
             self._download_playlist_dispatcher.dispatch(url, self, **kwargs)
-        except NoMatchException:
+        except _NoMatchException:
             log.e('[Error] Unsupported URL pattern with --playlist option.')
             exit(1)
 
+    def download_by_url(self, url, **kwargs):
+        if not kwargs.get('playlist') and self._download_playlist_dispatcher.test(url):
+            log.w('This is an album or drama. (use --playlist option to download all).')
+        else:
+            super().download_by_url(url, **kwargs)
+
+    def download(self, **kwargs):
+        kwargs['keep_obj'] = True   # keep the self.streams to download covers
+        super().download(**kwargs)
+        self.download_covers(self.title, self.streams, **kwargs)
+
     def extract(self, **kwargs):
         stream_id = kwargs.get('stream_id') or self.stream_types[0]['id']
         stream = self.streams[stream_id]
@@ -226,6 +276,10 @@ def url_sound_api(sound_id):
     def url_drama_api(drama_id):
         return 'https://www.missevan.com/dramaapi/getdrama?drama_id=' + str(drama_id)
 
+    @staticmethod
+    def url_danmaku_api(sound_id):
+        return 'https://www.missevan.com/sound/getdm?soundid=' + str(sound_id)
+
     @staticmethod
     def url_resource(uri):
         return 'https://static.missevan.com/' + uri

From 3a2b263c0007c9a70efdae94c835a5d5b36a416e Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Thu, 3 Oct 2019 21:32:23 +0800
Subject: [PATCH 0897/1225] [missevan] Fixed two bugs. * choose best quality
 available * download the nocover.png if there is no cover

---
 src/you_get/extractors/missevan.py | 37 ++++++++++++++++++++++--------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index c3cc82b9e1..1926eb6fd8 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -71,13 +71,14 @@ def dispatch(self, url, *args, **kwargs):
 
         raise _NoMatchException()
 
-
 missevan_stream_types = [
     {'id': '128bit', 'url_json_key': 'soundurl_128', 'desc': '128 Kbps'},
     {'id': '64bit', 'url_json_key': 'soundurl_64', 'desc': '64 Kbps'},
     {'id': '32bit', 'url_json_key': 'soundurl_32', 'desc': '32 Kbps'},
-    {'id': 'covers', 'url_json_key': 'cover_image', 'desc': '封面图'},
-    {'id': 'coversmini', 'url_json_key': 'cover_image', 'desc': '封面缩略图'}
+    {'id': 'covers', 'url_json_key': 'cover_image', 'desc': '封面图',
+     'default_src': 'http://static.missevan.com/covers/nocover.png'},
+    {'id': 'coversmini', 'url_json_key': 'cover_image', 'desc': '封面缩略图',
+     'default_src': 'http://static.missevan.com/coversmini/nocover.png'}
 ]
 
 def is_covers_stream(stream):
@@ -91,6 +92,13 @@ def get_file_extension(file_path, default=''):
         suffix = suffix[1:]
     return suffix or default
 
+def best_quality_stream_id(streams, stream_types):
+    for stream_type in stream_types:
+        if streams.get(stream_type['id']):
+            return stream_type['id']
+
+    raise AssertionError('no stream selected')
+
 
 class MissEvanWithStream(VideoExtractor):
 
@@ -165,12 +173,16 @@ def setup_streams(cls, sound):
             stream_id = stream_type['id']
             uri = sound[stream_type['url_json_key']]
             if is_covers_stream(stream_id):
-                resource_url = cls.url_resource(stream_id + '/' + uri)
+                if uri:
+                    resource_url = cls.url_resource(stream_id + '/' + uri)
+                else:
+                    resource_url = stream_type['default_src']
             else:
-                resource_url = cls.url_resource(uri)
+                resource_url = cls.url_resource(uri) if uri else None
 
-            container = get_file_extension(uri)
-            streams[stream_id] = {'src': [resource_url], 'container': container}
+            if resource_url:
+                container = get_file_extension(resource_url)
+                streams[stream_id] = {'src': [resource_url], 'container': container}
         return streams
 
     def prepare(self, **kwargs):
@@ -238,7 +250,9 @@ def download_drama(self, did, **kwargs):
 
     def download_playlist_by_url(self, url, **kwargs):
         # use the best quality by default
-        kwargs.setdefault('stream_id', self.stream_types[0]['id'])
+        if not kwargs.get('stream_id'):
+            stream_id = best_quality_stream_id(self.streams, self.stream_types)
+            kwargs['stream_id'] = stream_id
 
         self.url = url
         try:
@@ -254,12 +268,15 @@ def download_by_url(self, url, **kwargs):
             super().download_by_url(url, **kwargs)
 
     def download(self, **kwargs):
-        kwargs['keep_obj'] = True   # keep the self.streams to download covers
+        kwargs['keep_obj'] = True   # keep the self.streams to download cover
         super().download(**kwargs)
         self.download_covers(self.title, self.streams, **kwargs)
 
     def extract(self, **kwargs):
-        stream_id = kwargs.get('stream_id') or self.stream_types[0]['id']
+        stream_id = kwargs.get('stream_id')
+        if not stream_id:
+            stream_id = best_quality_stream_id(self.streams, self.stream_types)
+
         stream = self.streams[stream_id]
         if 'size' not in stream:
             stream['size'] = urls_size(stream['src'])

From 0ec71a84379347ff8491bb59588f7e93fe412e13 Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Fri, 4 Oct 2019 16:48:32 +0800
Subject: [PATCH 0898/1225] [missevan] Optimization * Change the stream_types
 and add the source stream_type * Improve support for --json/--info options *
 Fix download_playlist_by_url * Fix that open the cover with player (--player
 option)

---
 src/you_get/extractors/missevan.py | 74 +++++++++++++++++++-----------
 1 file changed, 46 insertions(+), 28 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 1926eb6fd8..cac6102bfe 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -26,7 +26,7 @@
 import os
 import re
 
-from ..common import get_content, urls_size, log
+from ..common import get_content, urls_size, log, player
 from ..extractor import VideoExtractor
 
 
@@ -72,15 +72,29 @@ def dispatch(self, url, *args, **kwargs):
         raise _NoMatchException()
 
 missevan_stream_types = [
-    {'id': '128bit', 'url_json_key': 'soundurl_128', 'desc': '128 Kbps'},
-    {'id': '64bit', 'url_json_key': 'soundurl_64', 'desc': '64 Kbps'},
-    {'id': '32bit', 'url_json_key': 'soundurl_32', 'desc': '32 Kbps'},
-    {'id': 'covers', 'url_json_key': 'cover_image', 'desc': '封面图',
-     'default_src': 'http://static.missevan.com/covers/nocover.png'},
-    {'id': 'coversmini', 'url_json_key': 'cover_image', 'desc': '封面缩略图',
-     'default_src': 'http://static.missevan.com/coversmini/nocover.png'}
+    {'id': 'source', 'quality': '源文件', 'url_json_key': 'soundurl',
+     'resource_url_fmt': 'sound/{resource_url}'},
+    {'id': '320', 'quality': '320 Kbps', 'url_json_key': 'soundurl_64'},
+    {'id': '128', 'quality': '128 Kbps', 'url_json_key': 'soundurl_128'},
+    {'id': '32', 'quality': '32 Kbps', 'url_json_key': 'soundurl_32'},
+    {'id': 'covers', 'desc': '封面图', 'url_json_key': 'cover_image',
+     'default_src': 'covers/nocover.png',
+     'resource_url_fmt': 'covers/{resource_url}'},
+    {'id': 'coversmini', 'desc': '封面缩略图', 'url_json_key': 'cover_image',
+     'default_src': 'coversmini/nocover.png',
+     'resource_url_fmt': 'coversmini/{resource_url}'}
 ]
 
+def _get_resource_uri(data, stream_type):
+    uri = data[stream_type['url_json_key']]
+    if not uri:
+        return stream_type.get('default_src')
+
+    uri_fmt = stream_type.get('resource_url_fmt')
+    if not uri_fmt:
+        return uri
+    return uri_fmt.format(resource_url=uri)
+
 def is_covers_stream(stream):
     stream = stream or ''
     return stream.lower() in ('covers', 'coversmini')
@@ -170,19 +184,16 @@ def setup_streams(cls, sound):
         streams = {}
 
         for stream_type in cls.stream_types:
-            stream_id = stream_type['id']
-            uri = sound[stream_type['url_json_key']]
-            if is_covers_stream(stream_id):
-                if uri:
-                    resource_url = cls.url_resource(stream_id + '/' + uri)
-                else:
-                    resource_url = stream_type['default_src']
-            else:
-                resource_url = cls.url_resource(uri) if uri else None
+            uri = _get_resource_uri(sound, stream_type)
+            resource_url = cls.url_resource(uri) if uri else None
 
             if resource_url:
                 container = get_file_extension(resource_url)
+                stream_id = stream_type['id']
                 streams[stream_id] = {'src': [resource_url], 'container': container}
+                quality = stream_type.get('quality')
+                if quality:
+                    streams[stream_id]['quality'] = quality
         return streams
 
     def prepare(self, **kwargs):
@@ -198,7 +209,10 @@ def prepare(self, **kwargs):
 
     @staticmethod
     def download_covers(title, streams, **kwargs):
-        if not is_covers_stream(kwargs.get('stream_id')):
+        if not is_covers_stream(kwargs.get('stream_id')) \
+                and not kwargs.get('json_output') \
+                and not kwargs.get('info_only') \
+                and not player:
             kwargs['stream_id'] = 'covers'
             MissEvanWithStream \
                 .create(title, streams) \
@@ -249,11 +263,6 @@ def download_drama(self, did, **kwargs):
             MissEvan().download_by_vid(sound_id, **kwargs)
 
     def download_playlist_by_url(self, url, **kwargs):
-        # use the best quality by default
-        if not kwargs.get('stream_id'):
-            stream_id = best_quality_stream_id(self.streams, self.stream_types)
-            kwargs['stream_id'] = stream_id
-
         self.url = url
         try:
             self._download_playlist_dispatcher.dispatch(url, self, **kwargs)
@@ -273,6 +282,13 @@ def download(self, **kwargs):
         self.download_covers(self.title, self.streams, **kwargs)
 
     def extract(self, **kwargs):
+        # fetch all streams size when output info or json
+        if kwargs.get('info_only') or kwargs.get('json_output'):
+            for _, stream in self.streams.items():
+                stream['size'] = urls_size(stream['src'])
+            return
+
+        # fetch size of the selected stream only
         stream_id = kwargs.get('stream_id')
         if not stream_id:
             stream_id = best_quality_stream_id(self.streams, self.stream_types)
@@ -283,23 +299,25 @@ def extract(self, **kwargs):
 
     @staticmethod
     def url_album_api(album_id):
-        return 'https://www.missevan.com/sound/soundalllist?albumid=' + str(album_id)
+        return f'https://www.missevan.com/sound' \
+               f'/soundalllist?albumid={album_id}'
 
     @staticmethod
     def url_sound_api(sound_id):
-        return 'https://www.missevan.com/sound/getsound?soundid=' + str(sound_id)
+        return f'https://www.missevan.com/sound/getsound?soundid={sound_id}'
 
     @staticmethod
     def url_drama_api(drama_id):
-        return 'https://www.missevan.com/dramaapi/getdrama?drama_id=' + str(drama_id)
+        return f'https://www.missevan.com/dramaapi' \
+               f'/getdrama?drama_id={drama_id}'
 
     @staticmethod
     def url_danmaku_api(sound_id):
-        return 'https://www.missevan.com/sound/getdm?soundid=' + str(sound_id)
+        return f'https://www.missevan.com/sound/getdm?soundid={sound_id}'
 
     @staticmethod
     def url_resource(uri):
-        return 'https://static.missevan.com/' + uri
+        return f'https://static.missevan.com/{uri}'
 
 site = MissEvan()
 site_info = 'MissEvan.com'

From 4add329e383b9e93350abf86ea883390564a062f Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Fri, 4 Oct 2019 18:01:53 +0800
Subject: [PATCH 0899/1225] [missevan] Prevent from being banned

---
 src/you_get/extractors/missevan.py | 46 ++++++++++++++++++++++--------
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index cac6102bfe..042ab6c0ff 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -29,6 +29,9 @@
 from ..common import get_content, urls_size, log, player
 from ..extractor import VideoExtractor
 
+_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 ' \
+       '(KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
+
 
 class _NoMatchException(Exception):
     pass
@@ -119,6 +122,11 @@ class MissEvanWithStream(VideoExtractor):
     name = 'MissEvan'
     stream_types = missevan_stream_types
 
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.referer = 'https://www.missevan.com/'
+        self.ua = _UA
+
     @classmethod
     def create(cls, title, streams, *, streams_sorted=None):
         obj = cls()
@@ -128,8 +136,8 @@ def create(cls, title, streams, *, streams_sorted=None):
         obj.streams_sorted.extend(streams_sorted)
         return obj
 
-    def fetch_danmaku(self, url, headers=None):
-        self.danmaku = get_content(url, headers or {})
+    def set_danmaku(self, danmaku):
+        self.danmaku = danmaku
         return self
 
     @staticmethod
@@ -164,18 +172,23 @@ class MissEvan(VideoExtractor):
     name = 'MissEvan'
     stream_types = missevan_stream_types
 
+    def __init__(self, *args):
+        super().__init__(*args)
+        self.referer = 'https://www.missevan.com/'
+        self.ua = _UA
+        self.__headers = {'User-Agent': self.ua, 'Referer': self.referer}
+
     __prepare_dispatcher = _Dispatcher()
 
     @__prepare_dispatcher.endpoint(
         re.compile(r'missevan\.com/sound/(?:player\?.*?id=)?(?P<sid>\d+)', re.I))
     def prepare_sound(self, sid, **kwargs):
-        content = get_content(self.url_sound_api(sid))
-        json_data = json.loads(content)
+        json_data = self._get_json(self.url_sound_api(sid))
         sound = json_data['info']['sound']
 
         self.title = sound['soundstr']
         if not is_covers_stream(kwargs.get('stream_id')):
-            self.danmaku = get_content(self.url_danmaku_api(sid))
+            self.danmaku = self._get_content(self.url_danmaku_api(sid))
 
         self.streams = self.setup_streams(sound)
 
@@ -223,8 +236,7 @@ def download_covers(title, streams, **kwargs):
     @_download_playlist_dispatcher.endpoint(
         re.compile(r'missevan\.com/album(?:info)?/(?P<aid>\d+)', re.I))
     def download_album(self, aid, **kwargs):
-        content = get_content(self.url_album_api(aid))
-        json_data = json.loads(content)
+        json_data = self._get_json(self.url_album_api(aid))
         album = json_data['info']['album']
         self.title = album['title']
         sounds = json_data['info']['sounds']
@@ -237,9 +249,10 @@ def download_album(self, aid, **kwargs):
             streams = self.setup_streams(sound)
             sound_id = sound['id']
             sound_title = sound['soundstr']
+            danmaku = self._get_content(self.url_danmaku_api(sound_id))
             MissEvanWithStream \
                 .create(sound_title, streams) \
-                .fetch_danmaku(self.url_danmaku_api(sound_id)) \
+                .set_danmaku(danmaku) \
                 .download(**kwargs)
 
             self.download_covers(sound_title, streams, **kwargs)
@@ -247,8 +260,7 @@ def download_album(self, aid, **kwargs):
     @_download_playlist_dispatcher.endpoint(
         re.compile(r'missevan\.com(?:/mdrama)?/drama/(?P<did>\d+)', re.I))
     def download_drama(self, did, **kwargs):
-        content = get_content(self.url_drama_api(did))
-        json_data = json.loads(content)
+        json_data = self._get_json(self.url_drama_api(did))
 
         drama = json_data['info']['drama']
 
@@ -282,14 +294,17 @@ def download(self, **kwargs):
         self.download_covers(self.title, self.streams, **kwargs)
 
     def extract(self, **kwargs):
+        stream_id = kwargs.get('stream_id')
+
         # fetch all streams size when output info or json
-        if kwargs.get('info_only') or kwargs.get('json_output'):
+        if kwargs.get('info_only') and not stream_id \
+                or kwargs.get('json_output'):
+
             for _, stream in self.streams.items():
                 stream['size'] = urls_size(stream['src'])
             return
 
         # fetch size of the selected stream only
-        stream_id = kwargs.get('stream_id')
         if not stream_id:
             stream_id = best_quality_stream_id(self.streams, self.stream_types)
 
@@ -297,6 +312,13 @@ def extract(self, **kwargs):
         if 'size' not in stream:
             stream['size'] = urls_size(stream['src'])
 
+    def _get_content(self, url):
+        return get_content(url, headers=self.__headers)
+
+    def _get_json(self, url):
+        content = self._get_content(url)
+        return json.loads(content)
+
     @staticmethod
     def url_album_api(album_id):
         return f'https://www.missevan.com/sound' \

From 0dbe1853511b1ee377b9c994a641e7fafcf5824e Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Fri, 4 Oct 2019 18:08:01 +0800
Subject: [PATCH 0900/1225] [missevan] Update README.md and add a test case

---
 README.md     | 1 +
 tests/test.py | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 360b5d0b25..0735bd8ade 100644
--- a/README.md
+++ b/README.md
@@ -421,6 +421,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 荔枝FM   | <http://www.lizhi.fm/>         | | |✓|
 | 秒拍     | <http://www.miaopai.com/>      |✓| | |
 | MioMio弹幕网 | <http://www.miomio.tv/>    |✓| | |
+| MissEvan<br/>猫耳FM | <http://www.missevan.com/> | | |✓|
 | 痞客邦   | <https://www.pixnet.net/>      |✓| | |
 | PPTV聚力 | <http://www.pptv.com/>         |✓| | |
 | 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
diff --git a/tests/test.py b/tests/test.py
index 9584ac51df..a652c8fbfb 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -6,8 +6,7 @@
     imgur,
     magisto,
     youtube,
-    bilibili,
-    toutiao,
+    missevan
 )
 
 
@@ -22,6 +21,11 @@ def test_magisto(self):
             info_only=True
         )
 
+    def test_missevan(self):
+        missevan.download('https://m.missevan.com/sound/1285995', info_only=True)
+        missevan.download_playlist(
+            'https://www.missevan.com/mdrama/drama/17579', info_only=True)
+
     def test_youtube(self):
         youtube.download(
             'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True

From 80443e3e8b3e0a2e1fc8835a4d74c0105b7972b5 Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Fri, 4 Oct 2019 19:01:47 +0800
Subject: [PATCH 0901/1225] [missevan] string compatible

---
 src/you_get/extractors/missevan.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 042ab6c0ff..0d57f7f373 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -321,25 +321,26 @@ def _get_json(self, url):
 
     @staticmethod
     def url_album_api(album_id):
-        return f'https://www.missevan.com/sound' \
-               f'/soundalllist?albumid={album_id}'
+        return 'https://www.missevan.com/sound' \
+               '/soundalllist?albumid=' + str(album_id)
 
     @staticmethod
     def url_sound_api(sound_id):
-        return f'https://www.missevan.com/sound/getsound?soundid={sound_id}'
+        return 'https://www.missevan.com/sound' \
+               '/getsound?soundid=' + str(sound_id)
 
     @staticmethod
     def url_drama_api(drama_id):
-        return f'https://www.missevan.com/dramaapi' \
-               f'/getdrama?drama_id={drama_id}'
+        return 'https://www.missevan.com/dramaapi' \
+               '/getdrama?drama_id=' + str(drama_id)
 
     @staticmethod
     def url_danmaku_api(sound_id):
-        return f'https://www.missevan.com/sound/getdm?soundid={sound_id}'
+        return 'https://www.missevan.com/sound/getdm?soundid=' + str(sound_id)
 
     @staticmethod
     def url_resource(uri):
-        return f'https://static.missevan.com/{uri}'
+        return 'https://static.missevan.com/' + uri
 
 site = MissEvan()
 site_info = 'MissEvan.com'

From 9a3c5a912d0c4da6869eebf7d3d5d268d7163194 Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Fri, 4 Oct 2019 21:02:36 +0800
Subject: [PATCH 0902/1225] [missevan test] Update test case

---
 tests/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index a652c8fbfb..edaa546fe9 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -24,7 +24,7 @@ def test_magisto(self):
     def test_missevan(self):
         missevan.download('https://m.missevan.com/sound/1285995', info_only=True)
         missevan.download_playlist(
-            'https://www.missevan.com/mdrama/drama/17579', info_only=True)
+            'https://www.missevan.com/albuminfo/287800', info_only=True)
 
     def test_youtube(self):
         youtube.download(

From 272442e36d6f5537d7e64c9f02a934e3731e854e Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Sat, 5 Oct 2019 16:10:50 +0800
Subject: [PATCH 0903/1225] [missevan] Show and skip paid resource

---
 src/you_get/extractors/missevan.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 0d57f7f373..63a2234232 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -187,6 +187,10 @@ def prepare_sound(self, sid, **kwargs):
         sound = json_data['info']['sound']
 
         self.title = sound['soundstr']
+        if sound.get('need_pay'):
+            log.e('付费资源无法下载')
+            return
+
         if not is_covers_stream(kwargs.get('stream_id')):
             self.danmaku = self._get_content(self.url_danmaku_api(sid))
 
@@ -246,9 +250,13 @@ def download_album(self, aid, **kwargs):
         kwargs['output_dir'] = output_dir
 
         for sound in sounds:
+            sound_title = sound['soundstr']
+            if sound.get('need_pay'):
+                log.w('跳过付费资源: ' + sound_title)
+                continue
+
             streams = self.setup_streams(sound)
             sound_id = sound['id']
-            sound_title = sound['soundstr']
             danmaku = self._get_content(self.url_danmaku_api(sound_id))
             MissEvanWithStream \
                 .create(sound_title, streams) \
@@ -263,6 +271,8 @@ def download_drama(self, did, **kwargs):
         json_data = self._get_json(self.url_drama_api(did))
 
         drama = json_data['info']['drama']
+        if drama.get('need_pay'):
+            log.w('该剧集包含付费资源, 付费资源将被跳过')
 
         self.title = drama['name']
         output_dir = os.path.abspath(kwargs.pop('output_dir', '.'))
@@ -271,6 +281,9 @@ def download_drama(self, did, **kwargs):
 
         episodes = json_data['info']['episodes']
         for each in episodes['episode']:
+            if each.get('need_pay'):
+                log.w('跳过付费资源: ' + each['soundstr'])
+                continue
             sound_id = each['sound_id']
             MissEvan().download_by_vid(sound_id, **kwargs)
 

From bbedf16d2d3a5e337bfd67d08dba5394d23eb28c Mon Sep 17 00:00:00 2001
From: WaferJay <474346873@qq.com>
Date: Sat, 5 Oct 2019 17:24:07 +0800
Subject: [PATCH 0904/1225] [missevan] Skip danmaku download during dry run

---
 src/you_get/extractors/missevan.py | 16 ++++++++--------
 tests/test.py                      |  4 +++-
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 63a2234232..c2b25a90d9 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -26,7 +26,7 @@
 import os
 import re
 
-from ..common import get_content, urls_size, log, player
+from ..common import get_content, urls_size, log, player, dry_run
 from ..extractor import VideoExtractor
 
 _UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 ' \
@@ -191,7 +191,7 @@ def prepare_sound(self, sid, **kwargs):
             log.e('付费资源无法下载')
             return
 
-        if not is_covers_stream(kwargs.get('stream_id')):
+        if not is_covers_stream(kwargs.get('stream_id')) and not dry_run:
             self.danmaku = self._get_content(self.url_danmaku_api(sid))
 
         self.streams = self.setup_streams(sound)
@@ -256,12 +256,12 @@ def download_album(self, aid, **kwargs):
                 continue
 
             streams = self.setup_streams(sound)
-            sound_id = sound['id']
-            danmaku = self._get_content(self.url_danmaku_api(sound_id))
-            MissEvanWithStream \
-                .create(sound_title, streams) \
-                .set_danmaku(danmaku) \
-                .download(**kwargs)
+            extractor = MissEvanWithStream.create(sound_title, streams)
+            if not dry_run:
+                sound_id = sound['id']
+                danmaku = self._get_content(self.url_danmaku_api(sound_id))
+                extractor.set_danmaku(danmaku)
+            extractor.download(**kwargs)
 
             self.download_covers(sound_title, streams, **kwargs)
 
diff --git a/tests/test.py b/tests/test.py
index edaa546fe9..e2f77a79b6 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -24,7 +24,9 @@ def test_magisto(self):
     def test_missevan(self):
         missevan.download('https://m.missevan.com/sound/1285995', info_only=True)
         missevan.download_playlist(
-            'https://www.missevan.com/albuminfo/287800', info_only=True)
+            'https://www.missevan.com/mdrama/drama/24130', info_only=True)
+        missevan.download_playlist(
+            'https://www.missevan.com/albuminfo/203090', info_only=True)
 
     def test_youtube(self):
         youtube.download(

From 27e10d56e5453dc4f65ce505abca245eac427c49 Mon Sep 17 00:00:00 2001
From: Jian Dai <daijian1@qq.com>
Date: Thu, 10 Oct 2019 10:11:05 +0800
Subject: [PATCH 0905/1225] Update qq.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

处理类似这样的微信公众号文章视频地址：https://v.qq.com/iframe/preview.html?width=500&amp;height=375&amp;auto=0&amp;vid=c0343icyvk6
---
 src/you_get/extractors/qq.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index fc4f3088cc..232a08b4ce 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -128,7 +128,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
     if 'mp.weixin.qq.com/s' in url:
         content = get_content(url)
-        vids = matchall(content, [r'\?vid=(\w+)'])
+        vids = matchall(content, [r'[?;]vid=(\w+)'])
         for vid in vids:
             qq_download_by_vid(vid, vid, output_dir, merge, info_only)
         return

From 968bfaab5ab1dd039953a83fb93da3d766de4328 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Thu, 17 Oct 2019 02:36:21 +0800
Subject: [PATCH 0906/1225] update bilibili.py for audio_base_url

---
 .gitignore                         |  6 +++
 src/you_get/extractors/bilibili.py | 71 ++++++++++++++++++------------
 2 files changed, 50 insertions(+), 27 deletions(-)

diff --git a/.gitignore b/.gitignore
index 0888e5ab95..99b18775ea 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,3 +83,9 @@ _*
 /.idea
 *.m4a
 *.DS_Store
+*.txt
+
+*.zip
+
+.vscode
+
diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 7eaf9b9fe0..e5ddbafcfe 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -123,7 +123,7 @@ def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
 
         try:
-            html_content = get_content(self.url, headers=self.bilibili_headers())
+            html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))
         except:
             html_content = ''  # live always returns 400 (why?)
         #self.title = match1(html_content,
@@ -255,17 +255,21 @@ def prepare(self, **kwargs):
                         size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
 
                         # find matching audio track
-                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
-                        for audio in playinfo['data']['dash']['audio']:
-                            if int(audio['id']) == audio_quality:
-                                audio_baseurl = audio['baseUrl']
-                                break
-                        if not audio_size_cache.get(audio_quality, False):
-                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
-                        size += audio_size_cache[audio_quality]
-
-                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+                        if playinfo['data']['dash']['audio']:
+                            audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                            for audio in playinfo['data']['dash']['audio']:
+                                if int(audio['id']) == audio_quality:
+                                    audio_baseurl = audio['baseUrl']
+                                    break
+                            if not audio_size_cache.get(audio_quality, False):
+                                audio_size_cache[audio_quality] = self.url_size(audio_baseurl, headers=self.bilibili_headers(referer=self.url))
+                            size += audio_size_cache[audio_quality]
+
+                            self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                            'src': [[baseurl], [audio_baseurl]], 'size': size}
+                        else:
+                            self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                            'src': [[baseurl]], 'size': size}
 
             # get danmaku
             self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
@@ -492,18 +496,22 @@ def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
                     size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))
 
                     # find matching audio track
-                    audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
-                    for audio in playinfo['data']['dash']['audio']:
-                        if int(audio['id']) == audio_quality:
-                            audio_baseurl = audio['baseUrl']
-                            break
-                    if not audio_size_cache.get(audio_quality, False):
-                        audio_size_cache[audio_quality] = self.url_size(audio_baseurl,
-                                                                        headers=self.bilibili_headers(referer=self.url))
-                    size += audio_size_cache[audio_quality]
-
-                    self.dash_streams[format_id] = {'container': container, 'quality': desc,
-                                                    'src': [[baseurl], [audio_baseurl]], 'size': size}
+                    if playinfo['data']['dash']['audio']:
+                        audio_baseurl = playinfo['data']['dash']['audio'][0]['baseUrl']
+                        for audio in playinfo['data']['dash']['audio']:
+                            if int(audio['id']) == audio_quality:
+                                audio_baseurl = audio['baseUrl']
+                                break
+                        if not audio_size_cache.get(audio_quality, False):
+                            audio_size_cache[audio_quality] = self.url_size(audio_baseurl,
+                                                                            headers=self.bilibili_headers(referer=self.url))
+                        size += audio_size_cache[audio_quality]
+
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl], [audio_baseurl]], 'size': size}
+                    else:
+                        self.dash_streams[format_id] = {'container': container, 'quality': desc,
+                                                        'src': [[baseurl]], 'size': size}
 
         # get danmaku
         self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
@@ -610,9 +618,18 @@ def download_playlist_by_url(self, url, **kwargs):
                                 self.extract(**kwargs)
                                 self.download(**kwargs)
             else:
-                for pi in range(1, pn + 1):
-                    purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
-                    self.__class__().download_by_url(purl, **kwargs)
+                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo = json.loads(playinfo_text) if playinfo_text else None
+
+                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
+                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
+                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+                for pi in range(pn):
+                    self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url)
+                    self.extract(**kwargs)
+                    self.download(**kwargs)
+                    # purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi+1)
+                    # self.__class__().download_by_url(purl, **kwargs)
 
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME

From 61b74e3ce94ecc726a9097998960274d8c0add0e Mon Sep 17 00:00:00 2001
From: sheerluck <sheer.luck.andrew@gmail.com>
Date: Thu, 24 Oct 2019 22:28:53 +0300
Subject: [PATCH 0907/1225] fix coub with quotes

for example, https://coub.com/view/xrbu1
---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5b16fc0490..2397a0a6b6 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -916,7 +916,7 @@ def get_output_filename(urls, title, ext, output_dir, merge, **kwargs):
     if kwargs.get('part', -1) >= 0:
         result = '%s[%02d]' % (result, kwargs.get('part'))
     result = '%s.%s' % (result, merged_ext)
-    return result
+    return result.replace("'", "_")
 
 def print_user_agent(faker=False):
     urllib_default_user_agent = 'Python-urllib/%d.%d' % sys.version_info[:2]

From 93fc74db2af9a6af02b574090e84c1c402099f50 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Mon, 28 Oct 2019 01:39:55 +0800
Subject: [PATCH 0908/1225] Update bilibili.py

download multipart video form given P number
---
 src/you_get/extractors/bilibili.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e5ddbafcfe..d0bbace3b6 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -624,7 +624,8 @@ def download_playlist_by_url(self, url, **kwargs):
                 html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
                 playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
                 playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-                for pi in range(pn):
+                p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or '1')-1
+                for pi in range(p,pn):
                     self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url)
                     self.extract(**kwargs)
                     self.download(**kwargs)

From bed017a7d6135a6aa223213c2942f1468266cf43 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Wed, 30 Oct 2019 00:52:54 +0800
Subject: [PATCH 0909/1225] Update bilibili.py

add refer
---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index d0bbace3b6..b70d8ba2ec 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -540,7 +540,7 @@ def download_playlist_by_url(self, url, **kwargs):
         self.url = url
         kwargs['playlist'] = True
 
-        html_content = get_content(self.url, headers=self.bilibili_headers())
+        html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))
 
         # sort it out
         if re.match(r'https?://(www\.)?bilibili\.com/bangumi/play/ep(\d+)', self.url):

From 80c4ebcee12d33175f5afa785cef967f90eeeee3 Mon Sep 17 00:00:00 2001
From: cage <120989324@qq.com>
Date: Wed, 13 Nov 2019 17:26:52 +0800
Subject: [PATCH 0910/1225] update bilibili.py

fix bug for flv only playlist video on bilibili
---
 src/you_get/extractors/bilibili.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index b70d8ba2ec..61331bba78 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -613,8 +613,12 @@ def download_playlist_by_url(self, url, **kwargs):
                         for choice in node_info['data']['edges']['choices']:
                             search_node_list.append(choice['node_id'])
                             if not choice['cid'] in download_cid_set:
-                                download_cid_set.add(choice['cid'] )
+                                download_cid_set.add(choice['cid'])
                                 self.prepare_by_cid(aid,choice['cid'],initial_state['videoData']['title']+('P{}. {}'.format(len(download_cid_set),choice['option'])),html_content,playinfo,playinfo_,url)
+                                try:
+                                    self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+                                except:
+                                    self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
                                 self.extract(**kwargs)
                                 self.download(**kwargs)
             else:
@@ -627,6 +631,10 @@ def download_playlist_by_url(self, url, **kwargs):
                 p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or '1')-1
                 for pi in range(p,pn):
                     self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url)
+                    try:
+                        self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
+                    except:
+                        self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
                     self.extract(**kwargs)
                     self.download(**kwargs)
                     # purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi+1)

From bd84c1241be7bc4eaa5d9fec0753906a6a90bbb0 Mon Sep 17 00:00:00 2001
From: Ciiiiing <2218999785@qq.com>
Date: Sun, 17 Nov 2019 17:55:19 +0800
Subject: [PATCH 0911/1225] bug fix, download bilibili VIP video, resolve
 1080P+ video

---
 src/you_get/extractors/bilibili.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e5ddbafcfe..6f5df454dc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -31,15 +31,17 @@ class Bilibili(VideoExtractor):
     ]
 
     @staticmethod
-    def height_to_quality(height):
-        if height <= 360:
+    def height_to_quality(height, qn):
+        if height <= 360 and qn <= 16:
             return 16
-        elif height <= 480:
+        elif height <= 480 and qn <= 32:
             return 32
-        elif height <= 720:
+        elif height <= 720 and qn <= 64:
             return 64
-        else:
+        elif height <= 1080 and qn <= 80:
             return 80
+        else:
+            return 112
 
     @staticmethod
     def bilibili_headers(referer=None, cookie=None):
@@ -144,7 +146,7 @@ def prepare(self, **kwargs):
             initial_state = json.loads(initial_state_text)
             ep_id = initial_state['epList'][0]['id']
             self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
-            html_content = get_content(self.url, headers=self.bilibili_headers())
+            html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))
 
         # sort it out
         if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
@@ -203,12 +205,12 @@ def prepare(self, **kwargs):
             if playinfo_ is not None:
                 playinfos.append(playinfo_)
             # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
+            for qn in [112, 80, 64, 32, 16]:
                 # automatic format for durl: qn=0
                 # for dash, qn does not matter
                 if current_quality is None or qn < current_quality:
                     api_url = self.bilibili_api(avid, cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
                     api_playinfo = json.loads(api_content)
                     if api_playinfo['code'] == 0:  # success
                         playinfos.append(api_playinfo)
@@ -216,7 +218,7 @@ def prepare(self, **kwargs):
                         message = api_playinfo['data']['message']
                 if best_quality is None or qn <= best_quality:
                     api_url = self.bilibili_interface_api(cid, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
                     api_playinfo_data = json.loads(api_content)
                     if api_playinfo_data.get('quality'):
                         playinfos.append({'code': 0, 'message': '0', 'ttl': 1, 'data': api_playinfo_data})
@@ -293,7 +295,7 @@ def prepare(self, **kwargs):
             cid = initial_state['epInfo']['cid']
             playinfos = []
             api_url = self.bilibili_bangumi_api(avid, cid, ep_id)
-            api_content = get_content(api_url, headers=self.bilibili_headers())
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
             api_playinfo = json.loads(api_content)
             if api_playinfo['code'] == 0:  # success
                 playinfos.append(api_playinfo)
@@ -302,12 +304,12 @@ def prepare(self, **kwargs):
                 return
             current_quality = api_playinfo['result']['quality']
             # get alternative formats from API
-            for qn in [80, 64, 32, 16]:
+            for qn in [112, 80, 64, 32, 16]:
                 # automatic format for durl: qn=0
                 # for dash, qn does not matter
                 if qn != current_quality:
                     api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers())
+                    api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
                     api_playinfo = json.loads(api_content)
                     if api_playinfo['code'] == 0:  # success
                         playinfos.append(api_playinfo)
@@ -329,7 +331,7 @@ def prepare(self, **kwargs):
                 if 'dash' in playinfo['result']:
                     for video in playinfo['result']['dash']['video']:
                         # playinfo['result']['quality'] does not reflect the correct quality of DASH stream
-                        quality = self.height_to_quality(video['height'])  # convert height to quality code
+                        quality = self.height_to_quality(video['height'], video['id'])  # convert height to quality code
                         s = self.stream_qualities[quality]
                         format_id = 'dash-' + s['id']  # prefix
                         container = 'mp4'  # enforce MP4 container

From b82d1be079b3eaa747f7c2f385aec8c4f34735f4 Mon Sep 17 00:00:00 2001
From: xiaohe <2754694716@qq.com>
Date: Wed, 20 Nov 2019 23:07:23 +0800
Subject: [PATCH 0912/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dacfun=E7=9A=84?=
 =?UTF-8?q?=E8=A7=86=E9=A2=91=E4=B8=8B=E8=BD=BD?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/acfun.py | 14 +++++++++++---
 tests/test.py                   |  5 ++++-
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 52fcb4f91e..6bf5964a72 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -121,9 +121,17 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         video_list = json_data.get('videoList')
         if len(video_list) > 1:
             title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
-
-        m3u8_url = json_data.get('currentVideoInfo')['playInfos'][0]['playUrls'][0]
-
+        currentVideoInfo = json_data.get('currentVideoInfo')
+        if 'playInfos' in currentVideoInfo:
+            m3u8_url = currentVideoInfo['playInfos'][0]['playUrls'][0]
+        elif 'ksPlayJson' in currentVideoInfo:
+            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] ) 
+            representation = ksPlayJson.get('adaptationSet').get('representation')
+            reps = []
+            for one in representation:
+                reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
+            m3u8_url = max(reps)[1]
+            
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
         html = get_content(url, headers=fake_headers)
         tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
diff --git a/tests/test.py b/tests/test.py
index e2f77a79b6..1674172257 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -6,7 +6,8 @@
     imgur,
     magisto,
     youtube,
-    missevan
+    missevan,
+    acfun
 )
 
 
@@ -38,6 +39,8 @@ def test_youtube(self):
             info_only=True
         )
 
+    def test_acfun(self):
+        acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)  
 
 if __name__ == '__main__':
     unittest.main()

From 867b66effc3554273efcb3e756ae5e388a1caa5c Mon Sep 17 00:00:00 2001
From: SFMDI <36741818+SFMDI@users.noreply.github.com>
Date: Sat, 23 Nov 2019 03:05:21 +0900
Subject: [PATCH 0913/1225] fix extractor naver.py

can download both old and recent videos without wrong result
---
 src/you_get/extractors/naver.py | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/naver.py b/src/you_get/extractors/naver.py
index add884e9bf..42a607e4eb 100644
--- a/src/you_get/extractors/naver.py
+++ b/src/you_get/extractors/naver.py
@@ -16,15 +16,8 @@ def naver_download_by_url(url, output_dir='.', merge=True, info_only=False, **kw
     ep = 'https://apis.naver.com/rmcnmv/rmcnmv/vod/play/v2.0/{}?key={}'
     page = get_content(url)
     try:
-        temp = re.search(r"<meta\s+property=\"og:video:url\"\s+content='(.+?)'>", page)
-        if temp is not None:
-            og_video_url = temp.group(1)
-            params_dict = urllib.parse.parse_qs(urllib.parse.urlparse(og_video_url).query)
-            vid = params_dict['vid'][0]
-            key = params_dict['outKey'][0]
-        else:
-            vid = re.search(r"\"videoId\"\s*:\s*\"(.+?)\"", page).group(1)
-            key = re.search(r"\"inKey\"\s*:\s*\"(.+?)\"", page).group(1)
+        vid = re.search(r"\"videoId\"\s*:\s*\"(.+?)\"", page).group(1)
+        key = re.search(r"\"inKey\"\s*:\s*\"(.+?)\"", page).group(1)
         meta_str = get_content(ep.format(vid, key))
         meta_json = json.loads(meta_str)
         if 'errorCode' in meta_json:
@@ -38,7 +31,7 @@ def naver_download_by_url(url, output_dir='.', merge=True, info_only=False, **kw
         size = url_size(video_url)
         print_info(site_info, title, 'mp4', size)
         if not info_only:
-            download_urls([video_url], title, 'mp4', size, **kwargs)
+            download_urls([video_url], title, 'mp4', size, output_dir, **kwargs)
     except:
         universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
 

From 6c4fbd0651ad0b002864cf5f33ed3b0f28d59a53 Mon Sep 17 00:00:00 2001
From: SFMDI <36741818+SFMDI@users.noreply.github.com>
Date: Sat, 23 Nov 2019 03:53:32 +0900
Subject: [PATCH 0914/1225] add tv.kakao.com extractor

can download video from tv.kakao.com
and sort __init__ by extarctor name
---
 src/you_get/common.py              |  1 +
 src/you_get/extractors/__init__.py |  7 +++--
 src/you_get/extractors/kakao.py    | 50 ++++++++++++++++++++++++++++++
 3 files changed, 55 insertions(+), 3 deletions(-)
 create mode 100644 src/you_get/extractors/kakao.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2397a0a6b6..19474a755d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -66,6 +66,7 @@
     'iwara'            : 'iwara',
     'joy'              : 'joy',
     'kankanews'        : 'bilibili',
+    'kakao'            : 'kakao',
     'khanacademy'      : 'khan',
     'ku6'              : 'ku6',
     'kuaishou'         : 'kuaishou',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 2961f01522..5ed5264bf0 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -33,7 +33,10 @@
 from .iqilu import *
 from .iqiyi import *
 from .joy import *
+from .khan import *
 from .ku6 import *
+from .kakao import *
+from .kuaishou import *
 from .kugou import *
 from .kuwo import *
 from .le import *
@@ -62,6 +65,7 @@
 from .sohu import *
 from .soundcloud import *
 from .suntv import *
+from .ted import *
 from .theplatform import *
 from .tiktok import *
 from .tucao import *
@@ -81,9 +85,6 @@
 from .yixia import *
 from .youku import *
 from .youtube import *
-from .ted import *
-from .khan import *
 from .zhanqi import *
-from .kuaishou import *
 from .zhibo import *
 from .zhihu import *
diff --git a/src/you_get/extractors/kakao.py b/src/you_get/extractors/kakao.py
new file mode 100644
index 0000000000..4ec282e312
--- /dev/null
+++ b/src/you_get/extractors/kakao.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+from ..common import *
+from .universal import *
+
+__all__ = ['kakao_download']
+
+
+def kakao_download(url, output_dir='.', info_only=False,  **kwargs):
+    json_request_url = 'https://videofarm.daum.net/controller/api/closed/v1_2/IntegratedMovieData.json?vid={}'
+
+    # in this implementation playlist not supported so use url_without_playlist
+    # if want to support playlist need to change that
+    if re.search('playlistId', url):
+        url = re.search(r"(.+)\?.+?", url).group(1)
+
+    page = get_content(url)
+    try:
+        vid = re.search(r"<meta name=\"vid\" content=\"(.+)\">", page).group(1)
+        title = re.search(r"<meta name=\"title\" content=\"(.+)\">", page).group(1)
+
+        meta_str = get_content(json_request_url.format(vid))
+        meta_json = json.loads(meta_str)
+
+        standard_preset = meta_json['output_list']['standard_preset']
+        output_videos = meta_json['output_list']['output_list']
+        size = ''
+        if meta_json['svcname'] == 'smr_pip':
+            for v in output_videos:
+                if v['preset'] == 'mp4_PIP_SMR_480P':
+                    size = int(v['filesize'])
+                    break
+        else:
+            for v in output_videos:
+                if v['preset'] == standard_preset:
+                    size = int(v['filesize'])
+                    break
+
+        video_url = meta_json['location']['url']
+
+        print_info(site_info, title, 'mp4', size)
+        if not info_only:
+            download_urls([video_url], title, 'mp4', size, output_dir, **kwargs)
+    except:
+        universal_download(url, output_dir, merge=kwargs['merge'], info_only=info_only, **kwargs)
+
+
+site_info = "tv.kakao.com"
+download = kakao_download
+download_playlist = playlist_not_supported('kakao')

From e9d5cc0232df0e1593d60aed7ae256c048178e0f Mon Sep 17 00:00:00 2001
From: Jerry <isjerryxiao@outlook.com>
Date: Sun, 1 Dec 2019 23:49:34 +0800
Subject: [PATCH 0915/1225] Fix playback of VideoExtractor.dash_streams

---
 src/you_get/common.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2397a0a6b6..65063647a4 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -272,15 +272,21 @@ def matchall(text, patterns):
 def launch_player(player, urls):
     import subprocess
     import shlex
+    urls = list(urls)
+    for url in urls.copy():
+        if type(url) is list:
+            urls.extend(url)
+    urls = [url for url in urls if type(url) is str]
+    assert urls
     if (sys.version_info >= (3, 3)):
         import shutil
         exefile=shlex.split(player)[0]
         if shutil.which(exefile) is not None:
-            subprocess.call(shlex.split(player) + list(urls))
+            subprocess.call(shlex.split(player) + urls)
         else:
             log.wtf('[Failed] Cannot find player "%s"' % exefile)
     else:
-        subprocess.call(shlex.split(player) + list(urls))
+        subprocess.call(shlex.split(player) + urls)
 
 
 def parse_query_param(url, param):

From 44698a0f39d6b6c5e5fd1e5a8efffe8de3278519 Mon Sep 17 00:00:00 2001
From: out001a <545827465@qq.com>
Date: Sun, 15 Dec 2019 12:29:20 +0800
Subject: [PATCH 0916/1225] [ixigua] fix error 'video_id not found'

---
 src/you_get/extractors/ixigua.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 20e4561618..151107a632 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -5,6 +5,8 @@
 
 from ..common import *
 import random
+import requests
+import string
 import ctypes
 from json import loads
 
@@ -80,7 +82,23 @@ def tmp():
 
 def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
-    html = get_html(url, faker=True)
+    sess = requests.session()
+    html = sess.get(url, headers=headers).text
+    conf = loads(match1(html, r"window\.config = (.+);"))
+    if not conf:
+        log.e("Get window.config from url failed, url: {}".format(url))
+        return
+    verify_url = conf['prefix'] + conf['url'] + '?key=' + conf['key'] + '&psm=' + conf['psm'] \
+        + '&_signature=' + ''.join(random.sample(string.ascii_letters + string.digits, 31))
+    try:
+        ok = get_content(verify_url)
+    except Exception as e:
+        ok = e.msg
+    if ok != 'OK':
+        log.e("Verify failed, verify_url: {}, result: {}".format(verify_url, ok))
+        return
+    html = sess.get(url, headers=headers).text
+
     video_id = match1(html, r"\"vid\":\"([^\"]+)")
     title = match1(html, r"\"player__videoTitle\">.*?<h1.*?>(.*)<\/h1><\/div>")
     if not video_id:

From a54a9b36260f84bcd6f7fda3017a54dc9b5330fd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 25 Dec 2019 02:47:29 +0100
Subject: [PATCH 0917/1225] [baidu] fix tiebapic

---
 src/you_get/extractors/baidu.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 7914667e61..77e666b3a9 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -140,8 +140,8 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
                                   output_dir=output_dir, merge=False)
 
             items = re.findall(
-                r'//imgsrc.baidu.com/forum/w[^"]+/([^/"]+)', html)
-            urls = ['http://imgsrc.baidu.com/forum/pic/item/' + i
+                r'//tiebapic.baidu.com/forum/w[^"]+/([^/"]+)', html)
+            urls = ['http://tiebapic.baidu.com/forum/pic/item/' + i
                     for i in set(items)]
 
             # handle albums
@@ -151,7 +151,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             album_info = json.loads(get_content(album_url))
             for i in album_info['data']['pic_list']:
                 urls.append(
-                    'http://imgsrc.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
+                    'http://tiebapic.baidu.com/forum/pic/item/' + i['pic_id'] + '.jpg')
 
             ext = 'jpg'
             size = float('Inf')

From 654371e851d0c8b2f403ee81d29e41d3bed52e2b Mon Sep 17 00:00:00 2001
From: chonpsk <chonpskz@gmail.com>
Date: Fri, 27 Dec 2019 22:55:51 +0800
Subject: [PATCH 0918/1225] fix issue about KeyError:
 'url_encoded_fmt_stream_map'

---
 src/you_get/extractors/youtube.py | 82 ++++++++++++++++++++++++-------
 1 file changed, 64 insertions(+), 18 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 4483f8eb85..3606a3c614 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -220,7 +220,10 @@ def prepare(self, **kwargs):
                     stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                     #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
                 except:
-                    stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
+                    if 'url_encoded_fmt_stream_map' not in video_info:
+                        stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']
+                    else: 
+                        stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
                     if re.search('([^"]*/base\.js)"', video_page):
                         self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
                     else:
@@ -302,19 +305,35 @@ def prepare(self, **kwargs):
                 exit(0)
 
         for stream in stream_list:
-            metadata = parse.parse_qs(stream)
-            stream_itag = metadata['itag'][0]
-            self.streams[stream_itag] = {
-                'itag': metadata['itag'][0],
-                'url': metadata['url'][0],
-                'sig': metadata['sig'][0] if 'sig' in metadata else None,
-                's': metadata['s'][0] if 's' in metadata else None,
-                'quality': metadata['quality'][0] if 'quality' in metadata else None,
-                #'quality': metadata['quality_label'][0] if 'quality_label' in metadata else None,
-                'type': metadata['type'][0],
-                'mime': metadata['type'][0].split(';')[0],
-                'container': mime_to_container(metadata['type'][0].split(';')[0]),
-            }
+            if isinstance(stream, str):
+                metadata = parse.parse_qs(stream)
+                stream_itag = metadata['itag'][0]
+                self.streams[stream_itag] = {
+                    'itag': metadata['itag'][0],
+                    'url': metadata['url'][0],
+                    'sig': metadata['sig'][0] if 'sig' in metadata else None,
+                    's': metadata['s'][0] if 's' in metadata else None,
+                    'quality': metadata['quality'][0] if 'quality' in metadata else None,
+                    #'quality': metadata['quality_label'][0] if 'quality_label' in metadata else None,
+                    'type': metadata['type'][0],
+                    'mime': metadata['type'][0].split(';')[0],
+                    'container': mime_to_container(metadata['type'][0].split(';')[0]),
+                }
+            else:
+                stream_itag = stream['itag']
+                self.streams[stream_itag] = {
+                    'itag': stream['itag'],
+                    'url': stream['url'] if 'url' in stream else None,
+                    'sig': None,
+                    's': None,
+                    'quality': stream['quality'],
+                    'type': stream['mimeType'],
+                    'mime': stream['mimeType'].split(';')[0],
+                    'container': mime_to_container(stream['mimeType'].split(';')[0]),
+                }
+                if 'cipher' in stream:
+                    self.streams[stream_itag].update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
+                                                           for _ in stream['cipher'].split('&')]))
 
         # Prepare caption tracks
         try:
@@ -425,10 +444,37 @@ def prepare(self, **kwargs):
                                  for i in afmt.split('&')])
                            for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
             except:
-                streams = [dict([(i.split('=')[0],
-                                  parse.unquote(i.split('=')[1]))
-                                 for i in afmt.split('&')])
-                           for afmt in video_info['adaptive_fmts'][0].split(',')]
+                if 'adaptive_fmts' in video_info:
+                    streams = [dict([(i.split('=')[0],
+                                      parse.unquote(i.split('=')[1]))
+                                     for i in afmt.split('&')])
+                               for afmt in video_info['adaptive_fmts'][0].split(',')]
+                else:
+                    streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                    for stream in streams:
+                        if 'qualityLabel' in stream:
+                            stream['quality_label'] = stream['qualityLabel']
+                            del stream['qualityLabel']
+                        if 'width' in stream:
+                            stream['size'] = '{}x{}'.format(stream['width'], stream['height'])
+                            del stream['width']
+                            del stream['height']
+                        stream['type'] = stream['mimeType']
+                        stream['clen'] = stream['contentLength']
+                        stream['init'] = '{}-{}'.format(
+                            stream['initRange']['start'],
+                            stream['initRange']['end'])
+                        stream['index'] = '{}-{}'.format(
+                            stream['indexRange']['start'],
+                            stream['indexRange']['end'])
+                        del stream['mimeType']
+                        del stream['contentLength']
+                        del stream['initRange']
+                        del stream['indexRange']
+                        if 'cipher' in stream:
+                            stream.update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
+                                                for _ in stream['cipher'].split('&')]))
+                            del stream['cipher']
 
             for stream in streams: # get over speed limiting
                 stream['url'] += '&ratebypass=yes'

From ab4bce79f3641a651f5ad2953e05edbe17df958f Mon Sep 17 00:00:00 2001
From: chonpsk <chonpskz@gmail.com>
Date: Fri, 27 Dec 2019 23:25:59 +0800
Subject: [PATCH 0919/1225] fix issue on itag

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 3606a3c614..ebb42c69dd 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -322,7 +322,7 @@ def prepare(self, **kwargs):
             else:
                 stream_itag = stream['itag']
                 self.streams[stream_itag] = {
-                    'itag': stream['itag'],
+                    'itag': str(stream['itag']),
                     'url': stream['url'] if 'url' in stream else None,
                     'sig': None,
                     's': None,
@@ -452,6 +452,7 @@ def prepare(self, **kwargs):
                 else:
                     streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
                     for stream in streams:
+                        stream['itag'] = str(stream['itag'])
                         if 'qualityLabel' in stream:
                             stream['quality_label'] = stream['qualityLabel']
                             del stream['qualityLabel']

From a934dea8c5d9e91b62f0a7b91a5da9a201f72982 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 28 Dec 2019 21:35:25 +0100
Subject: [PATCH 0920/1225] version 0.4.1388

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 1d87177c19..235b8f855c 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1355'
+__version__ = '0.4.1388'

From 767339915b44172dcfb3a394feed4af169f739fb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 6 Jan 2020 18:25:43 +0100
Subject: [PATCH 0921/1225] [tests] remove one test_imgur case since it fails
 too often

---
 tests/test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 1674172257..220b216994 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -14,7 +14,6 @@
 class YouGetTests(unittest.TestCase):
     def test_imgur(self):
         imgur.download('http://imgur.com/WVLk5nD', info_only=True)
-        imgur.download('http://imgur.com/gallery/WVLk5nD', info_only=True)
 
     def test_magisto(self):
         magisto.download(
@@ -40,7 +39,7 @@ def test_youtube(self):
         )
 
     def test_acfun(self):
-        acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)  
+        acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
 if __name__ == '__main__':
     unittest.main()

From b96acaa526f61667518ad0aac233a50eed9b38f4 Mon Sep 17 00:00:00 2001
From: laiqing <laiqing@staff.sina.com.cn>
Date: Wed, 8 Jan 2020 14:05:57 +0800
Subject: [PATCH 0922/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=90=9C=E7=8B=90?=
 =?UTF-8?q?=E5=8F=B7=E7=9A=84=E8=A7=86=E9=A2=91=E4=B8=8D=E8=83=BD=E4=B8=8B?=
 =?UTF-8?q?=E8=BD=BD=E7=9A=84=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/sohu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index a1afc1265c..1aedb3e613 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -26,7 +26,7 @@ def sohu_download(url, output_dir='.', merge=True, info_only=False, extractor_pr
         vid = r1('id=(\d+)', url)
     else:
         html = get_html(url)
-        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html)
+        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) or r1(r'bid:\'(\d+)\',', html)
     assert vid
 
     if extractor_proxy:

From 67c240abd043e5effd92f955d420f18e25c76dc2 Mon Sep 17 00:00:00 2001
From: shanhm <shanhm@tangdou.com>
Date: Wed, 8 Jan 2020 15:20:27 +0800
Subject: [PATCH 0923/1225] use urllib instead of requests

---
 src/you_get/extractors/ixigua.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 151107a632..2f11e7f9d8 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -5,10 +5,10 @@
 
 from ..common import *
 import random
-import requests
 import string
 import ctypes
 from json import loads
+from urllib import request
 
 __all__ = ['ixigua_download', 'ixigua_download_playlist_by_url']
 
@@ -82,8 +82,14 @@ def tmp():
 
 def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
-    sess = requests.session()
-    html = sess.get(url, headers=headers).text
+    resp = urlopen_with_retry(request.Request(url))
+    html = resp.read().decode('utf-8')
+
+    _cookies = []
+    for c in resp.getheader('Set-Cookie').split("httponly,"):
+        _cookies.append(c.strip().split(' ')[0])
+    headers['cookie'] = ' '.join(_cookies)
+
     conf = loads(match1(html, r"window\.config = (.+);"))
     if not conf:
         log.e("Get window.config from url failed, url: {}".format(url))
@@ -97,7 +103,7 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if ok != 'OK':
         log.e("Verify failed, verify_url: {}, result: {}".format(verify_url, ok))
         return
-    html = sess.get(url, headers=headers).text
+    html = get_content(url, headers=headers)
 
     video_id = match1(html, r"\"vid\":\"([^\"]+)")
     title = match1(html, r"\"player__videoTitle\">.*?<h1.*?>(.*)<\/h1><\/div>")

From 5943fb6ca34371c3a87219c7c67b5eb139b34980 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Jan 2020 17:36:16 +0100
Subject: [PATCH 0924/1225] [sohu] fix bid

---
 src/you_get/extractors/sohu.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 1aedb3e613..74374202e2 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -26,7 +26,7 @@ def sohu_download(url, output_dir='.', merge=True, info_only=False, extractor_pr
         vid = r1('id=(\d+)', url)
     else:
         html = get_html(url)
-        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) or r1(r'bid:\'(\d+)\',', html)
+        vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) or r1(r'bid:\'(\d+)\',', html) or r1(r'bid=(\d+)', html)
     assert vid
 
     if extractor_proxy:

From c3ae61c04e8235b444caedcd25064fa5af4f4c92 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 13 Jan 2020 22:16:33 +0100
Subject: [PATCH 0925/1225] [youtube] remove streams without contentLength (fix
 #2767)

---
 src/you_get/extractors/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ebb42c69dd..07c1382ef4 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -222,7 +222,7 @@ def prepare(self, **kwargs):
                 except:
                     if 'url_encoded_fmt_stream_map' not in video_info:
                         stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']
-                    else: 
+                    else:
                         stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
                     if re.search('([^"]*/base\.js)"', video_page):
                         self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
@@ -451,6 +451,8 @@ def prepare(self, **kwargs):
                                for afmt in video_info['adaptive_fmts'][0].split(',')]
                 else:
                     streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                    # streams without contentLength got broken urls, just remove them (#2767)
+                    streams = [stream for stream in streams if 'contentLength' in stream]
                     for stream in streams:
                         stream['itag'] = str(stream['itag'])
                         if 'qualityLabel' in stream:

From 5a008ad878fa2676ee4fa55a020f09acc7e5f66e Mon Sep 17 00:00:00 2001
From: Jarry Shaw <jarryshaw@icloud.com>
Date: Sat, 25 Jan 2020 12:11:41 +0800
Subject: [PATCH 0926/1225] Added PySocks extra requirement

---
 setup.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 21246c5f38..24dc9fb271 100755
--- a/setup.py
+++ b/setup.py
@@ -41,5 +41,9 @@
 
     classifiers = proj_info['classifiers'],
 
-    entry_points = {'console_scripts': proj_info['console_scripts']}
+    entry_points = {'console_scripts': proj_info['console_scripts']},
+
+    extras_require={
+        'socks': ['PySocks'],
+    }
 )

From ccdc58a82d17a0eebe81d4d337353b81d9b6cb68 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Jan 2020 00:19:51 +0100
Subject: [PATCH 0927/1225] [youtube] new pattern for function name

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 07c1382ef4..b8ca4280f6 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -94,7 +94,8 @@ def tr_js(code):
         f1 = match1(js, r'\.set\(\w+\.sp,encodeURIComponent\(([$\w]+)') or \
             match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
             match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
-            match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)')
+            match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)') or \
+            match1(js, r'=([$\w]+)\(decodeURIComponent\(')
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)

From 610c3e8942c9f7b5dad2b9342d869b4693a72ceb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 27 Jan 2020 00:27:15 +0100
Subject: [PATCH 0928/1225] version 0.4.1403

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 235b8f855c..c124a979de 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1388'
+__version__ = '0.4.1403'

From 771a89fc8568007b870c34834ed00f48038a0d2d Mon Sep 17 00:00:00 2001
From: Ok <venelin1@protonmail.com>
Date: Wed, 29 Jan 2020 01:25:21 +0200
Subject: [PATCH 0929/1225] fixed regex

---
 src/you_get/extractors/coub.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/coub.py b/src/you_get/extractors/coub.py
index 36a0a5d678..a71cbc18e5 100644
--- a/src/you_get/extractors/coub.py
+++ b/src/you_get/extractors/coub.py
@@ -79,7 +79,7 @@ def get_title_and_urls(json_data):
 
 
 def get_coub_data(html):
-    coub_data = r1(r'<script id=\'coubPageCoubJson\' type=\'text/json\'>([^<]+)</script>', html)
+    coub_data = r1(r'<script id=\'coubPageCoubJson\' type=\'text/json\'>([\w\W]+?(?=</script>))</script>', html)
     json_data = json.loads(coub_data)
     return json_data
 

From 5147481a89ea752913914ddd60366b8143b2a06c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 31 Jan 2020 15:11:58 +0100
Subject: [PATCH 0930/1225] [json_output] remove sort_keys in json.dumps call
 (fix #2773)

---
 src/you_get/json_output.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/json_output.py b/src/you_get/json_output.py
index 5971bd939b..c6195761fd 100644
--- a/src/you_get/json_output.py
+++ b/src/you_get/json_output.py
@@ -29,7 +29,7 @@ def output(video_extractor, pretty_print=True):
     if extra:
         out["extra"] = extra
     if pretty_print:
-        print(json.dumps(out, indent=4, sort_keys=True, ensure_ascii=False))
+        print(json.dumps(out, indent=4, ensure_ascii=False))
     else:
         print(json.dumps(out))
 

From cd9931e1f6b56f28dcb7202051bc9c6520b6e5ff Mon Sep 17 00:00:00 2001
From: nsb2006 <ucfresh@gmail.com>
Date: Mon, 10 Feb 2020 23:29:09 +0800
Subject: [PATCH 0931/1225] =?UTF-8?q?=E8=AE=BE=E6=83=B3=E4=BC=AA=E8=A3=85?=
 =?UTF-8?q?=E8=85=BE=E8=AE=AF=E8=A7=86=E9=A2=91=E5=AE=A2=E6=88=B7=E7=AB=AF?=
 =?UTF-8?q?=E4=B8=8B=E8=BD=BD1080P?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

尝试通过修改UA下载1080p，但是之前没接触过Python，依葫芦画瓢改了半天重新编译后发现UA还是默认的Python-urllib。

或者有没有UA的命令参数，试了-user-agent报错。

祝新春吉祥，百毒不侵，感谢。
---
 src/you_get/extractors/qq.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 232a08b4ce..6411b19552 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -6,6 +6,10 @@
 from .qie_video import download_by_url as qie_video_download
 from ..common import *
 
+headers = {
+    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)  QQLive/10275340/50192209 Chrome/43.0.2357.134 Safari/537.36 QBCore/3.43.561.202 QQBrowser/9.0.2524.400'
+}
+
 
 def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
 
@@ -14,7 +18,7 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
     platforms = [4100201, 11]
     for platform in platforms:
         info_api = 'http://vv.video.qq.com/getinfo?otype=json&appver=3.2.19.333&platform={}&defnpayver=1&defn=shd&vid={}'.format(platform, vid)
-        info = get_content(info_api)
+        info = get_content(info_api, headers)
         video_json = json.loads(match1(info, r'QZOutputJson=(.*)')[:-1])
         if not video_json.get('msg')=='cannot play outside':
             break
@@ -41,7 +45,7 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
             filename = '.'.join([fn_pre, magic_str, str(part), video_type])
 
         key_api = "http://vv.video.qq.com/getkey?otype=json&platform=11&format={}&vid={}&filename={}&appver=3.2.19.333".format(part_format_id, vid, filename)
-        part_info = get_content(key_api)
+        part_info = get_content(key_api, headers)
         key_json = json.loads(match1(part_info, r'QZOutputJson=(.*)')[:-1])
         if key_json.get('key') is None:
             vkey = video_json['vl']['vi'][0]['fvkey']
@@ -71,7 +75,7 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
     BASE_URL = 'http://cgi.kg.qq.com/fcgi-bin/kg_ugc_getdetail'
     params_str = '?dataType=jsonp&jsonp=callback&jsonpCallback=jsopgetsonginfo&v=4&outCharset=utf-8&shareid=' + shareid
     url = BASE_URL + params_str
-    content = get_content(url)
+    content = get_content(url, headers)
     json_str = content[len('jsonpcallback('):-1]
     json_data = json.loads(json_str)
 
@@ -127,7 +131,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         return
 
     if 'mp.weixin.qq.com/s' in url:
-        content = get_content(url)
+        content = get_content(url, headers)
         vids = matchall(content, [r'[?;]vid=(\w+)'])
         for vid in vids:
             qq_download_by_vid(vid, vid, output_dir, merge, info_only)
@@ -142,7 +146,7 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title=info_json['videoinfo']['title']
     elif 'kuaibao.qq.com' in url or re.match(r'http://daxue.qq.com/content/content/id/\d+', url):
         # http://daxue.qq.com/content/content/id/2321
-        content = get_content(url)
+        content = get_content(url, headers)
         vid = match1(content, r'vid\s*=\s*"\s*([^"]+)"')
         title = match1(content, r'title">([^"]+)</p>')
         title = title.strip() if title else vid
@@ -152,11 +156,11 @@ def qq_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         title = vid
     elif 'view.inews.qq.com' in url:
         # view.inews.qq.com/a/20180521V0Z9MH00
-        content = get_content(url)
+        content = get_content(url, headers)
         vid = match1(content, r'"vid":"(\w+)"')
         title = match1(content, r'"title":"(\w+)"')
     else:
-        content = get_content(url)
+        content = get_content(url, headers)
         #vid = parse_qs(urlparse(url).query).get('vid') #for links specified vid  like http://v.qq.com/cover/p/ps6mnfqyrfo7es3.html?vid=q0181hpdvo5
         rurl = match1(content, r'<link.*?rel\s*=\s*"canonical".*?href\s*="(.+?)".*?>') #https://v.qq.com/x/cover/9hpjiv5fhiyn86u/t0522x58xma.html
         vid = ""

From bf49e2d1b398d4901243115746b9fd14a71aceda Mon Sep 17 00:00:00 2001
From: flewsea <w48325832@gmail.com>
Date: Thu, 27 Feb 2020 22:38:31 +0800
Subject: [PATCH 0932/1225] =?UTF-8?q?=E6=94=AF=E6=8C=81=E4=B8=8B=E8=BD=BD?=
 =?UTF-8?q?=E9=A1=B5=E9=9D=A2=E5=86=85=E6=89=80=E6=9C=89=E8=A7=86=E9=A2=91?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/iwara.py | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index a30159d748..67a41d414c 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -9,12 +9,15 @@
     'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
     'Cache-Control': 'max-age=0',
-
     'Connection': 'keep-alive',
     'Save-Data': 'on',
     'Cookie':'has_js=1;show_adult=1',
 }
-
+stream_types = [
+        {'id': 'Source',      'container': 'mp4', 'video_profile': '原始'},
+        {'id': '540p',    'container': 'mp4', 'video_profile': '540p'},
+        {'id': '360p',   'container': 'mp4', 'video_profile': '360P'},
+    ]
 def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     global headers
     video_hash = match1(url, r'https?://\w+.iwara.tv/videos/(\w+)')
@@ -31,6 +34,17 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if not info_only:
         download_urls([down_urls], title, ext, size, output_dir, merge=merge, headers=headers)
 
+def download_playlist_by_url( url, **kwargs):
+    video_page = get_content(url)
+    # url_first=re.findall(r"(http[s]?://[^/]+)",url)
+    url_first=match1(url, r"(http[s]?://[^/]+)")
+    # print (url_first)
+    videos = set(re.findall(r'<a href="(/videos/[^"]+)"', video_page))
+    if(len(videos)>0):
+        for video in videos:
+            iwara_download(url_first+video, **kwargs)
+    else:
+        maybe_print('this page not found any videos')
 site_info = "Iwara"
 download = iwara_download
-download_playlist = playlist_not_supported('iwara')
+download_playlist = download_playlist_by_url

From 358d79778122c391d83b2eaed5c139be2f798e7f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 2 Mar 2020 14:27:30 +0100
Subject: [PATCH 0933/1225] [youtube] fix download for non-DASH streams

---
 src/you_get/extractors/youtube.py | 14 ++++++++++----
 tests/test.py                     |  3 +++
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index b8ca4280f6..38aa1a4e65 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -218,7 +218,10 @@ def prepare(self, **kwargs):
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
                     self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
                     # Workaround: get_video_info returns bad s. Why?
-                    stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
+                    if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
+                        stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
+                    else:
+                        stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                     #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
                 except:
                     if 'url_encoded_fmt_stream_map' not in video_info:
@@ -321,7 +324,7 @@ def prepare(self, **kwargs):
                     'container': mime_to_container(metadata['type'][0].split(';')[0]),
                 }
             else:
-                stream_itag = stream['itag']
+                stream_itag = str(stream['itag'])
                 self.streams[stream_itag] = {
                     'itag': str(stream['itag']),
                     'url': stream['url'] if 'url' in stream else None,
@@ -367,7 +370,7 @@ def prepare(self, **kwargs):
                 self.caption_tracks[lang] = srt
         except: pass
 
-        # Prepare DASH streams
+        # Prepare DASH streams (NOTE: not every video has DASH streams!)
         try:
             dashmpd = ytplayer_config['args']['dashmpd']
             dash_xml = parseString(get_content(dashmpd))
@@ -451,7 +454,10 @@ def prepare(self, **kwargs):
                                      for i in afmt.split('&')])
                                for afmt in video_info['adaptive_fmts'][0].split(',')]
                 else:
-                    streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                    try:
+                        streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                    except:  # no DASH stream at all
+                        return
                     # streams without contentLength got broken urls, just remove them (#2767)
                     streams = [stream for stream in streams if 'contentLength' in stream]
                     for stream in streams:
diff --git a/tests/test.py b/tests/test.py
index 220b216994..7187cfb066 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -37,6 +37,9 @@ def test_youtube(self):
             'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
             info_only=True
         )
+        youtube.download(
+            'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
+        )
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)

From 8b7566eeb3f82112ac9996619164503c8cd8f309 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 2 Mar 2020 14:45:57 +0100
Subject: [PATCH 0934/1225] version 0.4.1410

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index c124a979de..c867e5b871 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1403'
+__version__ = '0.4.1410'

From 11f78325e637cff01aad6e52d13f757052511965 Mon Sep 17 00:00:00 2001
From: Ivan Tham <pickfire@riseup.net>
Date: Sun, 8 Mar 2020 00:10:23 +0800
Subject: [PATCH 0935/1225] Sort return without duplicate condition

---
 src/you_get/util/log.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/you_get/util/log.py b/src/you_get/util/log.py
index 67b26b7815..81fd1bf548 100644
--- a/src/you_get/util/log.py
+++ b/src/you_get/util/log.py
@@ -99,6 +99,4 @@ def wtf(message, exit_code=1):
 
 def yes_or_no(message):
     ans = str(input('%s (y/N) ' % message)).lower().strip()
-    if ans == 'y':
-        return True
-    return False
+    return ans == 'y'

From 7a43ac0782325d596e2372519faf033738d9a9ea Mon Sep 17 00:00:00 2001
From: helong0911 <helong0911@hotmail.com>
Date: Sat, 14 Mar 2020 13:33:34 +0800
Subject: [PATCH 0936/1225] [baomihua] fix download 403

---
 src/you_get/extractors/baomihua.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/baomihua.py b/src/you_get/extractors/baomihua.py
index 99dd7132ef..9e97879a07 100644
--- a/src/you_get/extractors/baomihua.py
+++ b/src/you_get/extractors/baomihua.py
@@ -6,6 +6,16 @@
 
 import urllib
 
+def baomihua_headers(referer=None, cookie=None):
+	# a reasonable UA
+	ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'
+	headers = {'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': ua}
+	if referer is not None:
+		headers.update({'Referer': referer})
+	if cookie is not None:
+		headers.update({'Cookie': cookie})
+	return headers
+	
 def baomihua_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html('http://play.baomihua.com/getvideourl.aspx?flvid=%s&devicetype=phone_app' % id)
     host = r1(r'host=([^&]*)', html)
@@ -16,10 +26,10 @@ def baomihua_download_by_id(id, title=None, output_dir='.', merge=True, info_onl
     assert vid
     dir_str = r1(r'&dir=([^&]*)', html).strip()
     url = "http://%s/%s/%s.%s" % (host, dir_str, vid, type)
-    _, ext, size = url_info(url)
+    _, ext, size = url_info(url, headers=baomihua_headers())
     print_info(site_info, title, type, size)
     if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge = merge)
+        download_urls([url], title, ext, size, output_dir, merge = merge, headers=baomihua_headers())
 
 def baomihua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)

From 8a47a729a9805032a94b7ce5171609ef3b5cb90d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 19 Mar 2020 11:46:44 +0100
Subject: [PATCH 0937/1225] [tests] remove test_missevan

---
 tests/test.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 7187cfb066..b5454fbfe0 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -21,13 +21,6 @@ def test_magisto(self):
             info_only=True
         )
 
-    def test_missevan(self):
-        missevan.download('https://m.missevan.com/sound/1285995', info_only=True)
-        missevan.download_playlist(
-            'https://www.missevan.com/mdrama/drama/24130', info_only=True)
-        missevan.download_playlist(
-            'https://www.missevan.com/albuminfo/203090', info_only=True)
-
     def test_youtube(self):
         youtube.download(
             'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True

From a8d5819417ce0d4c7d7c4789043b2fa7e065b721 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 19 Mar 2020 11:57:36 +0100
Subject: [PATCH 0938/1225] update .travis.yml

---
 .travis.yml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8433fe75d3..8dd26bfab0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,15 +4,10 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
+  - "3.7"
+  - "3.8"
+  - "nightly"
   - "pypy3"
-matrix:
-  include:
-    - python: "3.7"
-      dist: xenial
-    - python: "3.8-dev"
-      dist: xenial
-    - python: "nightly"
-      dist: xenial
 before_install:
   - pip install flake8
 before_script:

From 50318b1e4d94da2e7034080cfa428feff3904df6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 19 Mar 2020 12:04:32 +0100
Subject: [PATCH 0939/1225] update .travis.yml (remove nightly)

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 8dd26bfab0..eedbeeb283 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,7 +6,7 @@ python:
   - "3.6"
   - "3.7"
   - "3.8"
-  - "nightly"
+  #- "nightly"  (flake8 not working in python 3.9 yet, module 'ast' has no attribute 'AugLoad')
   - "pypy3"
 before_install:
   - pip install flake8

From cfa93fb16c2f0460caf62f6fce6fada683dad564 Mon Sep 17 00:00:00 2001
From: Yiyin Gu <yiyingu0512@gmail.com>
Date: Fri, 20 Mar 2020 19:16:41 -0400
Subject: [PATCH 0940/1225] temp fix of netease download

Some vip download can download through this, others can't. Still looking into it.
---
 src/you_get/extractors/netease.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index f74747b189..c7c0f66683 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -107,6 +107,9 @@ def netease_video_download(vinfo, output_dir='.', info_only=False):
 
 def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix=""):
     title = "%s%s. %s" % (playlist_prefix, song['position'], song['name'])
+    url_best = "http://music.163.com/song/media/outer/url?id=" + \
+        str(song['id']) + ".mp3"
+    '''
     songNet = 'p' + song['mp3Url'].split('/')[2][1:]
 
     if 'hMusic' in song and song['hMusic'] != None:
@@ -115,7 +118,7 @@ def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix
         url_best = song['mp3Url']
     elif 'bMusic' in song:
         url_best = make_url(songNet, song['bMusic']['dfsId'])
-
+    '''
     netease_download_common(title, url_best,
                             output_dir=output_dir, info_only=info_only)
 

From b4ea5976fb51233e0289196eb39fa73a6f3e1829 Mon Sep 17 00:00:00 2001
From: brainbush <960821@gmail.com>
Date: Mon, 23 Mar 2020 15:52:15 +0800
Subject: [PATCH 0941/1225] add support for BVID of bilibili

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 444ccb6f13..29ba0aff2b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -159,7 +159,7 @@ def prepare(self, **kwargs):
             sort = 'live'
         elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
             sort = 'vc'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|(BV(\S+)))', self.url):
             sort = 'video'
         else:
             self.download_playlist_by_url(self.url, **kwargs)

From 25c481cdcddf40b784c4b24fd8840d1574854845 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 Mar 2020 11:55:48 +0100
Subject: [PATCH 0942/1225] purge dead sites

---
 README.md                             |  5 ---
 src/you_get/extractors/musicplayon.py | 38 -----------------------
 src/you_get/extractors/videomega.py   | 44 ---------------------------
 src/you_get/extractors/vidto.py       | 40 ------------------------
 4 files changed, 127 deletions(-)
 delete mode 100644 src/you_get/extractors/musicplayon.py
 delete mode 100644 src/you_get/extractors/videomega.py
 delete mode 100644 src/you_get/extractors/vidto.py

diff --git a/README.md b/README.md
index 0735bd8ade..3105766b53 100644
--- a/README.md
+++ b/README.md
@@ -368,15 +368,12 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | VK          | <http://vk.com/>              |✓|✓| |
 | Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
-| Vidto       | <http://vidto.me/>            |✓| | |
-| Videomega   | <http://videomega.tv/>        |✓| | |
 | Veoh        | <http://www.veoh.com/>        |✓| | |
 | **Tumblr**  | <https://www.tumblr.com/>     |✓|✓|✓|
 | TED         | <http://www.ted.com/>         |✓| | |
 | SoundCloud  | <https://soundcloud.com/>     | | |✓|
 | SHOWROOM    | <https://www.showroom-live.com/> |✓| | |
 | Pinterest   | <https://www.pinterest.com/>  | |✓| |
-| MusicPlayOn | <http://en.musicplayon.com/>  |✓| | |
 | MTV81       | <http://www.mtv81.com/>       |✓| | |
 | Mixcloud    | <https://www.mixcloud.com/>   | | |✓|
 | Metacafe    | <http://www.metacafe.com/>    |✓| | |
@@ -387,7 +384,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | InfoQ       | <http://www.infoq.com/presentations/> |✓| | |
 | Imgur       | <http://imgur.com/>           | |✓| |
 | Heavy Music Archive | <http://www.heavy-music.ru/> | | |✓|
-| **Google+** | <https://plus.google.com/>    |✓|✓| |
 | Freesound   | <http://www.freesound.org/>   | | |✓|
 | Flickr      | <https://www.flickr.com/>     |✓|✓| |
 | FC2 Video   | <http://video.fc2.com/>       |✓| | |
@@ -409,7 +405,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
 | 豆瓣     | <http://www.douban.com/>       |✓| |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
-| Panda<br/>熊猫 | <http://www.panda.tv/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
 | 风行网   | <http://www.fun.tv/>           |✓| | |
 | iQIYI<br/>爱奇艺 | <http://www.iqiyi.com/> |✓| | |
diff --git a/src/you_get/extractors/musicplayon.py b/src/you_get/extractors/musicplayon.py
deleted file mode 100644
index ffc4ec36a6..0000000000
--- a/src/you_get/extractors/musicplayon.py
+++ /dev/null
@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-
-from ..common import *
-from ..extractor import VideoExtractor
-
-import json
-
-class MusicPlayOn(VideoExtractor):
-    name = "MusicPlayOn"
-
-    stream_types = [
-        {'id': '720p HD'},
-        {'id': '360p SD'},
-    ]
-
-    def prepare(self, **kwargs):
-        content = get_content(self.url)
-
-        self.title = match1(content,
-                            r'setup\[\'title\'\] = "([^"]+)";')
-
-        for s in self.stream_types:
-            quality = s['id']
-            src = match1(content,
-                         r'src: "([^"]+)", "data-res": "%s"' % quality)
-            if src is not None:
-                url = 'http://en.musicplayon.com%s' % src
-                self.streams[quality] = {'url': url}
-
-    def extract(self, **kwargs):
-        for i in self.streams:
-            s = self.streams[i]
-            _, s['container'], s['size'] = url_info(s['url'])
-            s['src'] = [s['url']]
-
-site = MusicPlayOn()
-download = site.download_by_url
-# TBD: implement download_playlist
diff --git a/src/you_get/extractors/videomega.py b/src/you_get/extractors/videomega.py
deleted file mode 100644
index 34fb5205a1..0000000000
--- a/src/you_get/extractors/videomega.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['videomega_download']
-
-from ..common import *
-import ssl
-
-def videomega_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    # Hot-plug cookie handler
-    ssl_context = request.HTTPSHandler(
-        context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
-    cookie_handler = request.HTTPCookieProcessor()
-    opener = request.build_opener(ssl_context, cookie_handler)
-    opener.addheaders = [('Referer', url),
-                         ('Cookie', 'noadvtday=0')]
-    request.install_opener(opener)
-
-    if re.search(r'view\.php', url):
-        php_url = url
-    else:
-        content = get_content(url)
-        m = re.search(r'ref="([^"]*)";\s*width="([^"]*)";\s*height="([^"]*)"', content)
-        ref = m.group(1)
-        width, height = m.group(2), m.group(3)
-        php_url = 'http://videomega.tv/view.php?ref=%s&width=%s&height=%s' % (ref, width, height)
-    content = get_content(php_url)
-
-    title = match1(content, r'<title>(.*)</title>')
-    js = match1(content, r'(eval.*)')
-    t = match1(js, r'\$\("\w+"\)\.\w+\("\w+","([^"]+)"\)')
-    t = re.sub(r'(\w)', r'{\1}', t)
-    t = t.translate({87 + i: str(i) for i in range(10, 36)})
-    s = match1(js, r"'([^']+)'\.split").split('|')
-    src = t.format(*s)
-
-    type, ext, size = url_info(src, faker=True)
-
-    print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([src], title, ext, size, output_dir, merge=merge, faker=True)
-
-site_info = "Videomega.tv"
-download = videomega_download
-download_playlist = playlist_not_supported('videomega')
diff --git a/src/you_get/extractors/vidto.py b/src/you_get/extractors/vidto.py
deleted file mode 100644
index c4e3b87e8f..0000000000
--- a/src/you_get/extractors/vidto.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['vidto_download']
-
-from ..common import *
-import pdb
-import time
-
-
-def vidto_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_content(url)
-    params = {}
-    r = re.findall(
-        r'type="(?:hidden|submit)?"(?:.*?)name="(.+?)"\s* value="?(.+?)">', html)
-    for name, value in r:
-        params[name] = value
-    data = parse.urlencode(params).encode('utf-8')
-    req = request.Request(url)
-    print("Please wait for 6 seconds...")
-    time.sleep(6)
-    print("Starting")
-    new_html = request.urlopen(req, data).read().decode('utf-8', 'replace')
-    new_stff = re.search('lnk_download" href="(.*?)">', new_html)
-    if(new_stff):
-        url = new_stff.group(1)
-        title = params['fname']
-        type = ""
-        ext = ""
-        a, b, size = url_info(url)
-        print_info(site_info, title, type, size)
-        if not info_only:
-            download_urls([url], title, ext, size, output_dir, merge=merge)
-    else:
-        print("cannot find link, please review")
-        pdb.set_trace()
-
-
-site_info = "vidto.me"
-download = vidto_download
-download_playlist = playlist_not_supported('vidto')

From e1edd9f912c147a4f04d780a39169b14a589208c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 Mar 2020 12:03:41 +0100
Subject: [PATCH 0943/1225] purge dead sites

---
 src/you_get/common.py              | 3 ---
 src/you_get/extractors/__init__.py | 2 --
 2 files changed, 5 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 70602c89d8..8c609d8c37 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -83,7 +83,6 @@
     'missevan'         : 'missevan',
     'mixcloud'         : 'mixcloud',
     'mtv81'            : 'mtv81',
-    'musicplayon'      : 'musicplayon',
     'miaopai'          : 'yixia',
     'naver'            : 'naver',
     '7gogo'            : 'nanagogo',
@@ -107,8 +106,6 @@
     'twimg'            : 'twitter',
     'twitter'          : 'twitter',
     'ucas'             : 'ucas',
-    'videomega'        : 'videomega',
-    'vidto'            : 'vidto',
     'vimeo'            : 'vimeo',
     'wanmen'           : 'wanmen',
     'weibo'            : 'miaopai',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 5ed5264bf0..ce95904cba 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -49,7 +49,6 @@
 from .miomio import *
 from .mixcloud import *
 from .mtv81 import *
-from .musicplayon import *
 from .nanagogo import *
 from .naver import *
 from .netease import *
@@ -74,7 +73,6 @@
 from .twitter import *
 from .ucas import *
 from .veoh import *
-from .videomega import *
 from .vimeo import *
 from .vine import *
 from .vk import *

From 88d574a4e9144f5a122460a2ea24c849fd04b88b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 Mar 2020 12:13:33 +0100
Subject: [PATCH 0944/1225] version 0.4.1423

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index c867e5b871..ab19c2bee7 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1410'
+__version__ = '0.4.1423'

From 25422ea3c5520bafc35614865637968634f93086 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 Mar 2020 12:23:40 +0100
Subject: [PATCH 0945/1225] update supported Python versions

---
 you-get.json | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/you-get.json b/you-get.json
index 56f8212af2..e98e2e8a4a 100644
--- a/you-get.json
+++ b/you-get.json
@@ -18,14 +18,13 @@
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.0",
-    "Programming Language :: Python :: 3.1",
     "Programming Language :: Python :: 3.2",
     "Programming Language :: Python :: 3.3",
     "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Multimedia",

From f10a1cdade1398758aafa8137118d21e7b77179e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 23 Mar 2020 12:28:02 +0100
Subject: [PATCH 0946/1225] update LICENSE.txt

---
 LICENSE.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index 5964bf2038..a193d8e254 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,7 @@
 MIT License
 
-Copyright (c) 2012-2019 Mort Yao <mort.yao@gmail.com>
+Copyright (c) 2012-2020 Mort Yao <mort.yao@gmail.com> and other contributors
+              (https://github.com/soimort/you-get/graphs/contributors)
 Copyright (c) 2012 Boyu Guo <iambus@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

From d6afc2e829f152d3b6d88944d1ad1ce7fe30776b Mon Sep 17 00:00:00 2001
From: icpz <cc@icpz.dev>
Date: Tue, 24 Mar 2020 18:48:22 +0800
Subject: [PATCH 0947/1225] add support for BVID in playlist mode of bilibili

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 29ba0aff2b..045853f334 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -552,7 +552,7 @@ def download_playlist_by_url(self, url, **kwargs):
         elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
             sort = 'bangumi_md'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/av(\d+)', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|BV(\S+))', self.url):
             sort = 'video'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
             sort = 'space_channel'

From f12943aa00a5c59fe183eb9b59f6f1928d26a230 Mon Sep 17 00:00:00 2001
From: e <1160590998@qq.com>
Date: Sat, 28 Mar 2020 18:15:22 +0800
Subject: [PATCH 0948/1225] add support for BVID in watchlater mode.

---
 src/you_get/extractors/bilibili.py |  6 +++---
 tests/test.py                      | 10 +++++++++-
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 045853f334..c669415d79 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -132,10 +132,10 @@ def prepare(self, **kwargs):
         #                    r'<h1 title="([^"]+)"')
 
         # redirect: watchlater
-        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/av(\d+)', self.url):
-            avid = match1(self.url, r'/av(\d+)')
+        if re.match(r'https?://(www\.)?bilibili\.com/watchlater/#/(av(\d+)|BV(\S+)/?)', self.url):
+            avid = match1(self.url, r'/(av\d+)') or match1(self.url, r'/(BV\w+)')
             p = int(match1(self.url, r'/p(\d+)') or '1')
-            self.url = 'https://www.bilibili.com/video/av%s?p=%s' % (avid, p)
+            self.url = 'https://www.bilibili.com/video/%s?p=%s' % (avid, p)
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
         # redirect: bangumi/play/ss -> bangumi/play/ep
diff --git a/tests/test.py b/tests/test.py
index b5454fbfe0..6fd3db6c07 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -7,7 +7,8 @@
     magisto,
     youtube,
     missevan,
-    acfun
+    acfun,
+    bilibili
 )
 
 
@@ -37,5 +38,12 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
+    def test_bilibil(self):
+        bilibili.download(
+            "https://www.bilibili.com/watchlater/#/BV1PE411q7mZ/p6", info_only=True
+        )
+        bilibili.download(
+            "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
+        )
 if __name__ == '__main__':
     unittest.main()

From c7b7a996ffa348833787dd77da70be288c65a9a5 Mon Sep 17 00:00:00 2001
From: zhufengning <renhaihao@outlook.com>
Date: Sun, 29 Mar 2020 18:09:29 +0800
Subject: [PATCH 0949/1225] fix bilibili favlist download

and updated the api url
---
 src/you_get/extractors/bilibili.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index c669415d79..2152661fe3 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -103,8 +103,8 @@ def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
 
     @staticmethod
-    def bilibili_space_favlist_api(vmid, fid, pn=1, ps=100):
-        return 'https://api.bilibili.com/x/space/fav/arc?vmid=%s&fid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (vmid, fid, pn, ps)
+    def bilibili_space_favlist_api(fid, pn=1, ps=20):
+        return 'https://api.bilibili.com/x/v3/fav/resource/list?media_id=%s&pn=%s&ps=%s&order=mtime&type=0&tid=0&jsonp=jsonp' % (fid, pn, ps)
 
     @staticmethod
     def bilibili_space_video_api(mid, pn=1, ps=100):
@@ -679,20 +679,22 @@ def download_playlist_by_url(self, url, **kwargs):
         elif sort == 'space_favlist':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
             vmid, fid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_favlist_api(vmid, fid)
+            api_url = self.bilibili_space_favlist_api(fid)
             api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
             favlist_info = json.loads(api_content)
-            pc = favlist_info['data']['pagecount']
-
-            for pn in range(1, pc + 1):
-                api_url = self.bilibili_space_favlist_api(vmid, fid, pn=pn)
+            pc = favlist_info['data']['info']['media_count'] // len(favlist_info['data']['medias'])
+            if favlist_info['data']['info']['media_count'] % len(favlist_info['data']['medias']) != 0:
+                pc += 1
+            for pn in range(1, pc):
+                log.w('Extracting %s of %s pages ...' % (pn, pc))
+                api_url = self.bilibili_space_favlist_api(fid, pn=pn)
                 api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
                 favlist_info = json.loads(api_content)
 
-                epn, i = len(favlist_info['data']['archives']), 0
-                for video in favlist_info['data']['archives']:
+                epn, i = len(favlist_info['data']['medias']), 0
+                for video in favlist_info['data']['medias']:
                     i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
-                    url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                    url = 'https://www.bilibili.com/video/av%s' % video['id']
                     self.__class__().download_playlist_by_url(url, **kwargs)
 
         elif sort == 'space_video':

From d603266a421f6ee00f24a16cb29063403cee389a Mon Sep 17 00:00:00 2001
From: zhufengning <renhaihao@outlook.com>
Date: Sun, 29 Mar 2020 19:31:26 +0800
Subject: [PATCH 0950/1225] fix wrong range usage

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 2152661fe3..95ce707a01 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -685,7 +685,7 @@ def download_playlist_by_url(self, url, **kwargs):
             pc = favlist_info['data']['info']['media_count'] // len(favlist_info['data']['medias'])
             if favlist_info['data']['info']['media_count'] % len(favlist_info['data']['medias']) != 0:
                 pc += 1
-            for pn in range(1, pc):
+            for pn in range(1, pc + 1):
                 log.w('Extracting %s of %s pages ...' % (pn, pc))
                 api_url = self.bilibili_space_favlist_api(fid, pn=pn)
                 api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))

From 5c339cc68893fa67cdf2d09163e9c4ad1e85d060 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 29 Mar 2020 22:49:46 +0200
Subject: [PATCH 0951/1225] [baidu] support https

---
 src/you_get/extractors/baidu.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 77e666b3a9..521d5e996f 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -112,15 +112,15 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
             time.sleep(5)
             download_urls([real_url], title, ext, size,
                           output_dir, url, merge=merge, faker=True)
-    elif re.match(r'http://music.baidu.com/album/\d+', url):
-        id = r1(r'http://music.baidu.com/album/(\d+)', url)
+    elif re.match(r'https?://music.baidu.com/album/\d+', url):
+        id = r1(r'https?://music.baidu.com/album/(\d+)', url)
         baidu_download_album(id, output_dir, merge, info_only)
 
-    elif re.match('http://music.baidu.com/song/\d+', url):
-        id = r1(r'http://music.baidu.com/song/(\d+)', url)
+    elif re.match('https?://music.baidu.com/song/\d+', url):
+        id = r1(r'https?://music.baidu.com/song/(\d+)', url)
         baidu_download_song(id, output_dir, merge, info_only)
 
-    elif re.match('http://tieba.baidu.com/', url):
+    elif re.match('https?://tieba.baidu.com/', url):
         try:
             # embedded videos
             embed_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)

From b347b1bb06c2f2aee71ddb1d770d7c1294919cee Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 Mar 2020 02:40:05 +0200
Subject: [PATCH 0952/1225] [bilibili] support h

---
 src/you_get/extractors/bilibili.py | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 95ce707a01..f53af46883 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -28,6 +28,8 @@ class Bilibili(VideoExtractor):
          'container': 'FLV', 'video_resolution': '360p', 'desc': '流畅 360P'},
         # 'quality': 15?
         {'id': 'mp4', 'quality': 0},
+
+        {'id': 'jpg', 'quality': 0},
     ]
 
     @staticmethod
@@ -114,6 +116,10 @@ def bilibili_space_video_api(mid, pn=1, ps=100):
     def bilibili_vc_api(video_id):
         return 'https://api.vc.bilibili.com/clip/v1/video/detail?video_id=%s' % video_id
 
+    @staticmethod
+    def bilibili_h_api(doc_id):
+        return 'https://api.vc.bilibili.com/link_draw/v1/doc/detail?doc_id=%s' % doc_id
+
     @staticmethod
     def url_size(url, faker=False, headers={},err_value=0):
         try:
@@ -161,6 +167,8 @@ def prepare(self, **kwargs):
             sort = 'vc'
         elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|(BV(\S+)))', self.url):
             sort = 'video'
+        elif re.match(r'https?://h\.?bilibili\.com/(\d+)', self.url):
+            sort = 'h'
         else:
             self.download_playlist_by_url(self.url, **kwargs)
             return
@@ -426,6 +434,24 @@ def prepare(self, **kwargs):
             self.streams['mp4'] = {'container': container,
                                    'size': size, 'src': [playurl]}
 
+        # h images
+        elif sort == 'h':
+            m = re.match(r'https?://h\.?bilibili\.com/(\d+)', self.url)
+            doc_id = m.group(1)
+            api_url = self.bilibili_h_api(doc_id)
+            api_content = get_content(api_url, headers=self.bilibili_headers())
+            h_info = json.loads(api_content)
+
+            urls = []
+            for pic in h_info['data']['item']['pictures']:
+                img_src = pic['img_src']
+                urls.append(img_src)
+            size = urls_size(urls)
+
+            self.title = doc_id
+            container = 'jpg'  # enforce JPG container
+            self.streams[container] = {'container': container,
+                                       'size': size, 'src': urls}
 
     def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
         #response for interaction video

From 9858e2f25daca32f9205d5be9e3371e387976e2d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 Mar 2020 02:43:59 +0200
Subject: [PATCH 0953/1225] version 0.4.1432

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index ab19c2bee7..d5004187f2 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1423'
+__version__ = '0.4.1432'

From bd06317fcc947d2705adaae4ee5e2a21acececc4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 Mar 2020 18:16:58 +0200
Subject: [PATCH 0954/1225] [README] [bilibili] images and audios supported

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3105766b53..8ec210b735 100644
--- a/README.md
+++ b/README.md
@@ -402,7 +402,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **AcFun** | <http://www.acfun.cn/>        |✓| | |
 | **Baidu<br/>百度贴吧** | <http://tieba.baidu.com/> |✓|✓| |
 | 爆米花网 | <http://www.baomihua.com/>     |✓| | |
-| **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓| | |
+| **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓|✓|✓|
 | 豆瓣     | <http://www.douban.com/>       |✓| |✓|
 | 斗鱼     | <http://www.douyutv.com/>      |✓| | |
 | 凤凰视频 | <http://v.ifeng.com/>          |✓| | |

From 84a5611939443dbf03e9751a0b33598934647652 Mon Sep 17 00:00:00 2001
From: richard <yueyericardo@gmail.com>
Date: Tue, 7 Apr 2020 23:45:32 -0400
Subject: [PATCH 0955/1225] inital

---
 README.md                              |  1 +
 src/you_get/common.py                  |  1 +
 src/you_get/extractors/__init__.py     |  3 +-
 src/you_get/extractors/xinpianchang.py | 46 ++++++++++++++++++++++++++
 tests/test.py                          |  7 +++-
 5 files changed, 56 insertions(+), 2 deletions(-)
 create mode 100644 src/you_get/extractors/xinpianchang.py

diff --git a/README.md b/README.md
index 8ec210b735..3429f9d83c 100644
--- a/README.md
+++ b/README.md
@@ -436,6 +436,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 火猫TV   | <http://www.huomao.com/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
+| 新片场 | <https://www.xinpianchang.com//>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
 | TikTok | <https://www.tiktok.com/>      |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 8c609d8c37..2e4edef5b9 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -116,6 +116,7 @@
     'xiaokaxiu'        : 'yixia',
     'xiaojiadianvideo' : 'fc2video',
     'ximalaya'         : 'ximalaya',
+    'xinpianchang'     : 'xinpianchang',
     'yinyuetai'        : 'yinyuetai',
     'yizhibo'          : 'yizhibo',
     'youku'            : 'youku',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index ce95904cba..4280d236dd 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -79,10 +79,11 @@
 from .w56 import *
 from .wanmen import *
 from .xiami import *
+from .xinpianchang import *
 from .yinyuetai import *
 from .yixia import *
 from .youku import *
 from .youtube import *
 from .zhanqi import *
 from .zhibo import *
-from .zhihu import *
+from .zhihu import *
\ No newline at end of file
diff --git a/src/you_get/extractors/xinpianchang.py b/src/you_get/extractors/xinpianchang.py
new file mode 100644
index 0000000000..a15b193a7a
--- /dev/null
+++ b/src/you_get/extractors/xinpianchang.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+import re
+import json
+from ..extractor import VideoExtractor
+from ..common import get_content, playlist_not_supported
+
+
+class Xinpianchang(VideoExtractor):
+    stream_types = [
+        {'id': '4K', 'quality': '超清 4K', 'video_profile': 'mp4-4K'},
+        {'id': '2K', 'quality': '超清 2K', 'video_profile': 'mp4-2K'},
+        {'id': '1080', 'quality': '高清 1080P', 'video_profile': 'mp4-FHD'},
+        {'id': '720', 'quality': '高清 720P', 'video_profile': 'mp4-HD'},
+        {'id': '540', 'quality': '清晰 540P', 'video_profile': 'mp4-SD'},
+        {'id': '360', 'quality': '流畅 360P', 'video_profile': 'mp4-LD'}
+    ]
+
+    name = 'xinpianchang'
+
+    def prepare(self, **kwargs):
+        # find key
+        page_content = get_content(self.url)
+        match_rule = r"vid: \"(.+?)\","
+        key = re.findall(match_rule, page_content)[0]
+
+        # get videos info
+        video_url = 'https://openapi-vtom.vmovier.com/v3/video/' + key + '?expand=resource'
+        data = json.loads(get_content(video_url))
+        self.title = data["data"]["video"]["title"]
+        video_info = data["data"]["resource"]["progressive"]
+
+        # set streams dict
+        for video in video_info:
+            url = video["https_url"]
+            size = video["filesize"]
+            profile = video["profile_code"]
+            stype = [st for st in self.__class__.stream_types if st['video_profile'] == profile][0]
+
+            stream_data = dict(src=[url], size=size, container='mp4', quality=stype['quality'])
+            print(stream_data)
+            self.streams[stype['id']] = stream_data
+
+
+download = Xinpianchang().download_by_url
+download_playlist = playlist_not_supported('xinpianchang')
diff --git a/tests/test.py b/tests/test.py
index 6fd3db6c07..5bc0a2e50b 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -8,7 +8,8 @@
     youtube,
     missevan,
     acfun,
-    bilibili
+    bilibili,
+    xinpianchang
 )
 
 
@@ -45,5 +46,9 @@ def test_bilibil(self):
         bilibili.download(
             "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
         )
+
+    def test_xinpianchang(self):
+        imgur.download('https://www.xinpianchang.com/a10673220', info_only=True)
+
 if __name__ == '__main__':
     unittest.main()

From b771248d23a73c7dc18e2b1ea5bd13247342e456 Mon Sep 17 00:00:00 2001
From: richard <yueyericardo@gmail.com>
Date: Tue, 7 Apr 2020 23:54:43 -0400
Subject: [PATCH 0956/1225] fix

---
 src/you_get/extractors/xinpianchang.py | 1 -
 tests/test.py                          | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/xinpianchang.py b/src/you_get/extractors/xinpianchang.py
index a15b193a7a..48830d97af 100644
--- a/src/you_get/extractors/xinpianchang.py
+++ b/src/you_get/extractors/xinpianchang.py
@@ -38,7 +38,6 @@ def prepare(self, **kwargs):
             stype = [st for st in self.__class__.stream_types if st['video_profile'] == profile][0]
 
             stream_data = dict(src=[url], size=size, container='mp4', quality=stype['quality'])
-            print(stream_data)
             self.streams[stype['id']] = stream_data
 
 
diff --git a/tests/test.py b/tests/test.py
index 5bc0a2e50b..5e4de738dd 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -48,7 +48,7 @@ def test_bilibil(self):
         )
 
     def test_xinpianchang(self):
-        imgur.download('https://www.xinpianchang.com/a10673220', info_only=True)
+        xinpianchang.download('https://www.xinpianchang.com/a10673220', info_only=True)
 
 if __name__ == '__main__':
     unittest.main()

From 4e0ca6f3e4a02d851a51e56ebcff472891ad6a56 Mon Sep 17 00:00:00 2001
From: richard <yueyericardo@gmail.com>
Date: Tue, 7 Apr 2020 23:55:50 -0400
Subject: [PATCH 0957/1225] rm test

---
 tests/test.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 5e4de738dd..6fd3db6c07 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -8,8 +8,7 @@
     youtube,
     missevan,
     acfun,
-    bilibili,
-    xinpianchang
+    bilibili
 )
 
 
@@ -46,9 +45,5 @@ def test_bilibil(self):
         bilibili.download(
             "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
         )
-
-    def test_xinpianchang(self):
-        xinpianchang.download('https://www.xinpianchang.com/a10673220', info_only=True)
-
 if __name__ == '__main__':
     unittest.main()

From 018cfde6048707a8a642493a3dc0e934de2f267e Mon Sep 17 00:00:00 2001
From: Richard Xue <yueyericardo@gmail.com>
Date: Wed, 8 Apr 2020 00:08:44 -0400
Subject: [PATCH 0958/1225] Update xinpianchang.py

---
 src/you_get/extractors/xinpianchang.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/xinpianchang.py b/src/you_get/extractors/xinpianchang.py
index 48830d97af..fac3d01fd0 100644
--- a/src/you_get/extractors/xinpianchang.py
+++ b/src/you_get/extractors/xinpianchang.py
@@ -7,6 +7,7 @@
 
 
 class Xinpianchang(VideoExtractor):
+    name = 'xinpianchang'
     stream_types = [
         {'id': '4K', 'quality': '超清 4K', 'video_profile': 'mp4-4K'},
         {'id': '2K', 'quality': '超清 2K', 'video_profile': 'mp4-2K'},
@@ -16,8 +17,6 @@ class Xinpianchang(VideoExtractor):
         {'id': '360', 'quality': '流畅 360P', 'video_profile': 'mp4-LD'}
     ]
 
-    name = 'xinpianchang'
-
     def prepare(self, **kwargs):
         # find key
         page_content = get_content(self.url)

From c34c0af6a4bbc602cc48ab66a657a82da540b2d9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 12 Apr 2020 01:02:51 +0200
Subject: [PATCH 0959/1225] [universal] support site-relative path

---
 src/you_get/extractors/universal.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 8b9a24c954..4c3d34e913 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -99,6 +99,14 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         for rel_url in rel_urls:
             urls += [ r1(r'(.*/)', url) + rel_url ]
 
+        # site-relative path
+        rel_urls = []
+        rel_urls += re.findall(r'href="(/[^"]+\.jpe?g)"', page, re.I)
+        rel_urls += re.findall(r'href="(/[^"]+\.png)"', page, re.I)
+        rel_urls += re.findall(r'href="(/[^"]+\.gif)"', page, re.I)
+        for rel_url in rel_urls:
+            urls += [ r1(r'(https?://[^/]+)', url) + rel_url ]
+
         # MPEG-DASH MPD
         mpd_urls = re.findall(r'src="(https?://[^"]+\.mpd)"', page)
         for mpd_url in mpd_urls:

From 793778040add4b8fbf2944827d0118494ff79430 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 14 Apr 2020 23:14:52 +0200
Subject: [PATCH 0960/1225] [universal] sometimes naive

---
 src/you_get/extractors/universal.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 4c3d34e913..abc6947599 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -107,6 +107,11 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         for rel_url in rel_urls:
             urls += [ r1(r'(https?://[^/]+)', url) + rel_url ]
 
+        # sometimes naive
+        urls += re.findall(r'data-original="(https?://[^"]+\.jpe?g)"', page, re.I)
+        urls += re.findall(r'data-original="(https?://[^"]+\.png)"', page, re.I)
+        urls += re.findall(r'data-original="(https?://[^"]+\.gif)"', page, re.I)
+
         # MPEG-DASH MPD
         mpd_urls = re.findall(r'src="(https?://[^"]+\.mpd)"', page)
         for mpd_url in mpd_urls:

From 07417e6ef15de6236827ab2ecffaea59a19ad9cb Mon Sep 17 00:00:00 2001
From: daixian <amano_tooko@qq.com>
Date: Thu, 23 Apr 2020 23:43:53 +0800
Subject: [PATCH 0961/1225] [bilibili]support 4k

---
 src/you_get/extractors/bilibili.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f53af46883..f3316cd60e 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -10,6 +10,8 @@ class Bilibili(VideoExtractor):
 
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
+        {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '2160p', 'desc': '超清 4K'},
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '1080p', 'desc': '高清 1080P60'},
         {'id': 'hdflv2', 'quality': 112, 'audio_quality': 30280,
@@ -42,8 +44,10 @@ def height_to_quality(height, qn):
             return 64
         elif height <= 1080 and qn <= 80:
             return 80
-        else:
+        elif height <= 1080 and qn <= 112:
             return 112
+        else:
+            return 120
 
     @staticmethod
     def bilibili_headers(referer=None, cookie=None):
@@ -213,7 +217,7 @@ def prepare(self, **kwargs):
             if playinfo_ is not None:
                 playinfos.append(playinfo_)
             # get alternative formats from API
-            for qn in [112, 80, 64, 32, 16]:
+            for qn in [120, 112, 80, 64, 32, 16]:
                 # automatic format for durl: qn=0
                 # for dash, qn does not matter
                 if current_quality is None or qn < current_quality:
@@ -312,7 +316,7 @@ def prepare(self, **kwargs):
                 return
             current_quality = api_playinfo['result']['quality']
             # get alternative formats from API
-            for qn in [112, 80, 64, 32, 16]:
+            for qn in [120, 112, 80, 64, 32, 16]:
                 # automatic format for durl: qn=0
                 # for dash, qn does not matter
                 if qn != current_quality:

From eca7a1d569d0d94b09aa4889fffccf18f6a1755e Mon Sep 17 00:00:00 2001
From: jack-zheng <lanmolei0124@163.com>
Date: Tue, 5 May 2020 23:45:20 +0800
Subject: [PATCH 0962/1225] support https type downloading from host of
 tv.cctv.com

---
 src/you_get/extractors/cntv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/cntv.py b/src/you_get/extractors/cntv.py
index 479481defe..a56cde6f78 100644
--- a/src/you_get/extractors/cntv.py
+++ b/src/you_get/extractors/cntv.py
@@ -44,12 +44,12 @@ def cntv_download_by_id(rid, **kwargs):
 def cntv_download(url, **kwargs):
     if re.match(r'http://tv\.cntv\.cn/video/(\w+)/(\w+)', url):
         rid = match1(url, r'http://tv\.cntv\.cn/video/\w+/(\w+)')
-    elif re.match(r'http://tv\.cctv\.com/\d+/\d+/\d+/\w+.shtml', url):
+    elif re.match(r'http(s)?://tv\.cctv\.com/\d+/\d+/\d+/\w+.shtml', url):
         rid = r1(r'var guid = "(\w+)"', get_content(url))
     elif re.match(r'http://\w+\.cntv\.cn/(\w+/\w+/(classpage/video/)?)?\d+/\d+\.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/(\w+/)*VIDE\d+.shtml', url) or \
          re.match(r'http://(\w+).cntv.cn/(\w+)/classpage/video/(\d+)/(\d+).shtml', url) or \
-         re.match(r'http://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
+         re.match(r'http(s)?://\w+.cctv.com/\d+/\d+/\d+/\w+.shtml', url) or \
          re.match(r'http://\w+.cntv.cn/\d+/\d+/\d+/\w+.shtml', url): 
         page = get_content(url)
         rid = r1(r'videoCenterId","(\w+)"', page)

From 1c7a0c946ec98acee0d753ed50f25fb054a5ba1a Mon Sep 17 00:00:00 2001
From: chrisww <chrisww@live.com>
Date: Wed, 13 May 2020 16:32:16 -0400
Subject: [PATCH 0963/1225] Fix resuming when downloading in chunked mode

---
 src/you_get/common.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2e4edef5b9..41d67cfc16 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -629,10 +629,12 @@ def url_save(
     if refer is not None:
         tmp_headers['Referer'] = refer
     if type(url) is list:
-        file_size = urls_size(url, faker=faker, headers=tmp_headers)
+        chunk_sizes = [url_size(url, faker=faker, headers=tmp_headers) for url in url]
+        file_size = sum(chunk_sizes)
         is_chunked, urls = True, url
     else:
         file_size = url_size(url, faker=faker, headers=tmp_headers)
+        chunk_sizes = [file_size]
         is_chunked, urls = False, [url]
 
     continue_renameing = True
@@ -696,9 +698,13 @@ def numreturn(a):
     else:
         open_mode = 'wb'
 
-    for url in urls:
+    chunk_start = 0
+    chunk_end = 0
+    for i, url in enumerate(urls):
         received_chunk = 0
-        if received < file_size:
+        chunk_start += 0 if i == 0 else chunk_sizes[i - 1]
+        chunk_end += chunk_sizes[i]
+        if received < file_size and received < chunk_end:
             if faker:
                 tmp_headers = fake_headers
             '''
@@ -708,8 +714,9 @@ def numreturn(a):
             else:
                 headers = {}
             '''
-            if received and not is_chunked:  # only request a range when not chunked
-                tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+            if received:
+                # chunk_start will always be 0 if not chunked
+                tmp_headers['Range'] = 'bytes=' + str(received - chunk_start) + '-'
             if refer:
                 tmp_headers['Referer'] = refer
 
@@ -757,8 +764,7 @@ def numreturn(a):
                         elif not is_chunked and received == file_size:  # Download finished
                             break
                         # Unexpected termination. Retry request
-                        if not is_chunked:  # when
-                            tmp_headers['Range'] = 'bytes=' + str(received) + '-'
+                        tmp_headers['Range'] = 'bytes=' + str(received - chunk_start) + '-'
                         response = urlopen_with_retry(
                             request.Request(url, headers=tmp_headers)
                         )

From 773b53f6d735f7cfdff55c234d2ff9af0094e9c8 Mon Sep 17 00:00:00 2001
From: chrisww <chrisww@live.com>
Date: Wed, 20 May 2020 22:46:04 -0400
Subject: [PATCH 0964/1225] Try to show non-dash formats for bangumi videos in
 bilibili

---
 src/you_get/extractors/bilibili.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f3316cd60e..94e5479f65 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -81,8 +81,8 @@ def bilibili_audio_menu_song_api(sid, ps=100):
         return 'https://www.bilibili.com/audio/music-service-c/web/song/of-menu?sid=%s&pn=1&ps=%s' % (sid, ps)
 
     @staticmethod
-    def bilibili_bangumi_api(avid, cid, ep_id, qn=0):
-        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=16' % (avid, cid, qn, ep_id)
+    def bilibili_bangumi_api(avid, cid, ep_id, qn=0, fnval=16):
+        return 'https://api.bilibili.com/pgc/player/web/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&ep_id=%s&fnver=0&fnval=%s' % (avid, cid, qn, ep_id, fnval)
 
     @staticmethod
     def bilibili_interface_api(cid, qn=0):
@@ -316,15 +316,16 @@ def prepare(self, **kwargs):
                 return
             current_quality = api_playinfo['result']['quality']
             # get alternative formats from API
-            for qn in [120, 112, 80, 64, 32, 16]:
-                # automatic format for durl: qn=0
-                # for dash, qn does not matter
-                if qn != current_quality:
-                    api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn)
-                    api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-                    api_playinfo = json.loads(api_content)
-                    if api_playinfo['code'] == 0:  # success
-                        playinfos.append(api_playinfo)
+            for fnval in [8, 16]:
+                for qn in [120, 112, 80, 64, 32, 16]:
+                    # automatic format for durl: qn=0
+                    # for dash, qn does not matter
+                    if qn != current_quality:
+                        api_url = self.bilibili_bangumi_api(avid, cid, ep_id, qn=qn, fnval=fnval)
+                        api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                        api_playinfo = json.loads(api_content)
+                        if api_playinfo['code'] == 0:  # success
+                            playinfos.append(api_playinfo)
 
             for playinfo in playinfos:
                 if 'durl' in playinfo['result']:

From 81ba2bc65c42d0eabd109eceffbb225f6819d0da Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 2 Jun 2020 16:31:09 +0200
Subject: [PATCH 0965/1225] [twitter] Twitter is obsoleting its legacy HTML
 (2020-06-01)

---
 src/you_get/extractors/twitter.py | 86 ++++++++++++-------------------
 1 file changed, 34 insertions(+), 52 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 4b239e679b..0ea5b1415f 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -41,58 +41,40 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         r1(r'<meta name="twitter:site:id" content="([^"]*)"', html)
     page_title = "{} [{}]".format(screen_name, item_id)
 
-    try: # extract images
-        urls = re.findall(r'property="og:image"\s*content="([^"]+:large)"', html)
-        assert urls
-        images = []
-        for url in urls:
-            url = ':'.join(url.split(':')[:-1]) + ':orig'
-            filename = parse.unquote(url.split('/')[-1])
-            title = '.'.join(filename.split('.')[:-1])
-            ext = url.split(':')[-2].split('.')[-1]
-            size = int(get_head(url)['Content-Length'])
-            images.append({'title': title,
-                           'url': url,
-                           'ext': ext,
-                           'size': size})
-        size = sum([image['size'] for image in images])
-        print_info(site_info, page_title, images[0]['ext'], size)
-
-        if not info_only:
-            for image in images:
-                title = image['title']
-                ext = image['ext']
-                size = image['size']
-                url = image['url']
-                print_info(site_info, title, ext, size)
-                download_urls([url], title, ext, size,
-                              output_dir=output_dir)
-
-    except: # extract video
-        #i_url = 'https://twitter.com/i/videos/' + item_id
-        #i_content = get_content(i_url)
-        #js_url = r1(r'src="([^"]+)"', i_content)
-        #js_content = get_content(js_url)
-        #authorization = r1(r'"(Bearer [^"]+)"', js_content)
-        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
-
-        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
-        ga_content = post_content(ga_url, headers={'authorization': authorization})
-        guest_token = json.loads(ga_content)['guest_token']
-
-        api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
-        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
-
-        info = json.loads(api_content)
-        variants = info['globalObjects']['tweets'][item_id]['extended_entities']['media'][0]['video_info']['variants']
-        variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
-        urls = [ variants[-1]['url'] ]
-        size = urls_size(urls)
-        mime, ext = variants[-1]['content_type'], 'mp4'
-
-        print_info(site_info, page_title, mime, size)
-        if not info_only:
-            download_urls(urls, page_title, ext, size, output_dir, merge=merge)
+    authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+
+    ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
+    ga_content = post_content(ga_url, headers={'authorization': authorization})
+    guest_token = json.loads(ga_content)['guest_token']
+
+    api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
+    api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
+
+    info = json.loads(api_content)
+    media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+    for medium in media:
+        if 'video_info' in medium:
+            # FIXME: we're assuming one tweet only contains one video here
+            variants = medium['video_info']['variants']
+            variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
+            urls = [ variants[-1]['url'] ]
+            size = urls_size(urls)
+            mime, ext = variants[-1]['content_type'], 'mp4'
+
+            print_info(site_info, page_title, mime, size)
+            if not info_only:
+                download_urls(urls, page_title, ext, size, output_dir, merge=merge)
+
+        else:
+            title = item_id + '_' + medium['media_url_https'].split('.')[-2].split('/')[-1]
+            urls = [ medium['media_url_https'] + ':orig' ]
+            size = urls_size(urls)
+            ext = medium['media_url_https'].split('.')[-1]
+
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls(urls, title, ext, size, output_dir, merge=merge)
+
 
 site_info = "Twitter.com"
 download = twitter_download

From c81cfb58dbf9fc8c61be25e3fa1d07cf22914316 Mon Sep 17 00:00:00 2001
From: TOCK Chiu <tock.chiu@gmail.com>
Date: Wed, 3 Jun 2020 00:26:18 +0800
Subject: [PATCH 0966/1225] [youtube] fix `KeyError: 'url'` on some videos

---
 src/you_get/extractors/youtube.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 38aa1a4e65..f8630b8c1c 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -335,9 +335,9 @@ def prepare(self, **kwargs):
                     'mime': stream['mimeType'].split(';')[0],
                     'container': mime_to_container(stream['mimeType'].split(';')[0]),
                 }
-                if 'cipher' in stream:
+                if 'signatureCipher' in stream:
                     self.streams[stream_itag].update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
-                                                           for _ in stream['cipher'].split('&')]))
+                                                           for _ in stream['signatureCipher'].split('&')]))
 
         # Prepare caption tracks
         try:
@@ -481,10 +481,10 @@ def prepare(self, **kwargs):
                         del stream['contentLength']
                         del stream['initRange']
                         del stream['indexRange']
-                        if 'cipher' in stream:
+                        if 'signatureCipher' in stream:
                             stream.update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
-                                                for _ in stream['cipher'].split('&')]))
-                            del stream['cipher']
+                                                for _ in stream['signatureCipher'].split('&')]))
+                            del stream['signatureCipher']
 
             for stream in streams: # get over speed limiting
                 stream['url'] += '&ratebypass=yes'

From 9b6c53d07747b913aad77c5abf6ef040cab06f2e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 13 Jun 2020 22:21:39 +0200
Subject: [PATCH 0967/1225] [tiktok] fix contentUrl extraction

---
 src/you_get/extractors/tiktok.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 2c388054d8..632e44c2d5 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -9,7 +9,7 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     title = r1(r'<title.*?>(.*?)</title>', html)
     video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
     title = '%s [%s]' % (title, video_id)
-    source = r1(r'<video .*?src="([^"]+)"', html)
+    source = r1(r'<video .*?src="([^"]+)"', html) or r1(r'"contentUrl":"([^"]+)"', html)
     mime, ext, size = url_info(source)
 
     print_info(site_info, title, mime, size)

From 5b9b0d878c802c794f7d8e0ac57187e648caee78 Mon Sep 17 00:00:00 2001
From: QingQiz <sofeeys@outlook.com>
Date: Sat, 11 Jul 2020 20:01:34 +0800
Subject: [PATCH 0968/1225] fix: soundcloud: HTTP Error 401: Unauthorized feat:
 soundcloud download playlist

---
 src/you_get/common.py                |  14 ++++
 src/you_get/extractors/soundcloud.py | 106 ++++++++++++++++++---------
 src/you_get/processor/ffmpeg.py      |  12 +++
 tests/test.py                        |  14 +++-
 4 files changed, 110 insertions(+), 36 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2e4edef5b9..b7fd61ffed 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1064,6 +1064,20 @@ def download_urls(
                 for part in parts:
                     os.remove(part)
 
+        elif ext == 'mp3':
+            try:
+                from .processor.ffmpeg import has_ffmpeg_installed
+
+                assert has_ffmpeg_installed()
+                from .processor.ffmpeg import ffmpeg_concat_mp3_to_mp3
+                ffmpeg_concat_mp3_to_mp3(parts, output_filepath)
+                print('Merged into %s' % output_filename)
+            except:
+                raise
+            else:
+                for part in parts:
+                    os.remove(part)
+
         else:
             print("Can't merge %s files" % ext)
 
diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
index 1a4061ffd0..ecd3fc8d7e 100644
--- a/src/you_get/extractors/soundcloud.py
+++ b/src/you_get/extractors/soundcloud.py
@@ -1,44 +1,80 @@
 #!/usr/bin/env python
 
-__all__ = ['soundcloud_download', 'soundcloud_download_by_id']
+__all__ = ['sndcd_download']
 
 from ..common import *
+import re
 import json
 import urllib.error
 
-client_id = 'WKcQQdEZw7Oi01KqtHWxeVSxNyRzgT8M'
-
-def soundcloud_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False):
-    assert title
-    url = 'https://api.soundcloud.com/tracks/{}/{}?client_id={}'.format(id, 'stream', client_id)
-    
-    type, ext, size = url_info(url)
-    
-    print_info(site_info, title, type, size)
-
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge = merge)
-
-def soundcloud_i1_api(track_id):
-    url = 'https://api.soundcloud.com/i1/tracks/{}/streams?client_id={}'.format(track_id, client_id)
-    return json.loads(get_content(url))['http_mp3_128_url']
-
-def soundcloud_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    url = 'https://api.soundcloud.com/resolve.json?url={}&client_id={}'.format(url, client_id)
-    metadata = get_content(url)
-    info = json.loads(metadata)
-    title = info["title"]
-    real_url = info.get('download_url')
-    if real_url is None:
-        real_url = info.get('steram_url')
-    if real_url is None:
-        raise Exception('Cannot get media URI for {}'.format(url))
-    real_url = soundcloud_i1_api(info['id'])
-    mime, ext, size = url_info(real_url)
-    print_info(site_info, title, mime, size)
-    if not info_only:
-        download_urls([real_url], title, ext, size, output_dir, merge=merge)
+
+def get_sndcd_apikey():
+    home_page = get_content('https://soundcloud.com')
+    js_url = re.findall(r'script crossorigin src="(.+?)"></script>', home_page)[-1]
+
+    client_id = get_content(js_url)
+    return re.search(r'client_id:"(.+?)"', client_id).group(1)
+
+
+def get_resource_info(resource_url, client_id):
+    cont = get_content(resource_url, decoded=True)
+
+    x = re.escape('forEach(function(e){n(e)})}catch(t){}})},')
+    x = re.search(r'' + x + r'(.*)\);</script>', cont)
+
+    info = json.loads(x.group(1))[-1]['data'][0]
+
+    info = info['tracks'] if info.get('track_count') else [info]
+
+    ids = [i['id'] for i in info if i.get('comment_count') is None]
+    ids = list(map(str, ids))
+    ids_split = ['%2C'.join(ids[i:i+10]) for i in range(0, len(ids), 10)]
+    api_url = 'https://api-v2.soundcloud.com/tracks?ids={ids}&client_id={client_id}&%5Bobject%20Object%5D=&app_version=1584348206&app_locale=en'
+
+    res = []
+    for ids in ids_split:
+        uri = api_url.format(ids=ids, client_id=client_id)
+        cont = get_content(uri, decoded=True)
+        res += json.loads(cont)
+
+    res = iter(res)
+    info = [next(res) if i.get('comment_count') is None else i for i in info]
+
+    return info
+
+
+def sndcd_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    client_id = get_sndcd_apikey()
+
+    r_info = get_resource_info(url, client_id)
+
+    for info in r_info:
+        title = info['title']
+        metadata = info.get('publisher_metadata')
+
+        transcodings = info['media']['transcodings']
+        sq = [i for i in transcodings if i['quality'] == 'sq']
+        hq = [i for i in transcodings if i['quality'] == 'hq']
+        # source url
+        surl = sq[0] if hq == [] else hq[0]
+        surl = surl['url']
+
+        uri = surl + '?client_id=' + client_id
+        r = get_content(uri)
+        surl = json.loads(r)['url']
+
+        m3u8 = get_content(surl)
+        # url list
+        urll = re.findall(r'http.*?(?=\n)', m3u8)
+
+        size = urls_size(urll)
+        print_info(site_info, title, 'audio/mpeg', size)
+        print(end='', flush=True)
+
+        if not info_only:
+            download_urls(urll, title=title, ext='mp3', total_size=size, output_dir=output_dir, merge=True)
+
 
 site_info = "SoundCloud.com"
-download = soundcloud_download
-download_playlist = playlist_not_supported('soundcloud')
+download = sndcd_download
+download_playlist = sndcd_download
diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 63679b834d..de78d5250b 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -180,6 +180,18 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     else:
         raise
 
+def ffmpeg_concat_mp3_to_mp3(files, output='output.mp3'):
+    print('Merging video parts... ', end="", flush=True)
+
+    files = 'concat:' + '|'.join(files)
+
+    params = [FFMPEG] + LOGLEVEL + ['-y']
+    params += ['-i', files, '-acodec', 'copy', output]
+
+    subprocess.call(params)
+
+    return True
+
 def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     print('Merging video parts... ', end="", flush=True)
     # Use concat demuxer on FFmpeg >= 1.1
diff --git a/tests/test.py b/tests/test.py
index 6fd3db6c07..38db15ddc4 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -8,7 +8,8 @@
     youtube,
     missevan,
     acfun,
-    bilibili
+    bilibili,
+    soundcloud
 )
 
 
@@ -45,5 +46,16 @@ def test_bilibil(self):
         bilibili.download(
             "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
         )
+
+    def test_soundcloud(self):
+        ## single song
+        soundcloud.download(
+            'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
+        )
+        ## playlist
+        soundcloud.download(
+            'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
+        )
+
 if __name__ == '__main__':
     unittest.main()

From 0cf81580058043f2d4cc3dc214769cb3ad71a7a8 Mon Sep 17 00:00:00 2001
From: Karthikeyan Singaravelan <tir.karthi@gmail.com>
Date: Wed, 15 Jul 2020 12:41:40 +0000
Subject: [PATCH 0969/1225] Use ElementTree instead of deprecated cElementTree.

---
 src/you_get/extractors/ckplayer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/ckplayer.py b/src/you_get/extractors/ckplayer.py
index 9115989796..5ff1f7b1fa 100644
--- a/src/you_get/extractors/ckplayer.py
+++ b/src/you_get/extractors/ckplayer.py
@@ -6,7 +6,7 @@
 
 __all__ = ['ckplayer_download']
 
-from xml.etree import cElementTree as ET
+from xml.etree import ElementTree as ET
 from copy import copy
 from ..common import *
 #----------------------------------------------------------------------

From f878b4fb9a2dee9d3fdc7d75fa357b5a59d33972 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 18 Jul 2020 18:09:25 +0200
Subject: [PATCH 0970/1225] [acfun] change bangumi URL first (see #2820)

---
 src/you_get/extractors/acfun.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 6bf5964a72..b83c28594d 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -125,14 +125,14 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         if 'playInfos' in currentVideoInfo:
             m3u8_url = currentVideoInfo['playInfos'][0]['playUrls'][0]
         elif 'ksPlayJson' in currentVideoInfo:
-            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] ) 
+            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
             representation = ksPlayJson.get('adaptationSet').get('representation')
             reps = []
             for one in representation:
                 reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
             m3u8_url = max(reps)[1]
-            
-    elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/ab(\d+)", url):
+
+    elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", url):
         html = get_content(url, headers=fake_headers)
         tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
         json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]

From 74fa92b2d7757cb024d6128156dc81e99f2fcbce Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 18 Jul 2020 18:44:18 +0200
Subject: [PATCH 0971/1225] [twitter] download media from quoted tweet

---
 src/you_get/extractors/twitter.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 0ea5b1415f..602c18f65e 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -51,7 +51,30 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
 
     info = json.loads(api_content)
-    media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+    if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+        # if the tweet contains media, download them
+        media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+
+    elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
+        # if the tweet does not contain media, but it quotes a tweet
+        # and the quoted tweet contains media, download them
+        item_id = info['globalObjects']['tweets'][item_id]['quoted_status_id_str']
+
+        api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
+        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
+
+        info = json.loads(api_content)
+
+        if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+            media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+        else:
+            # quoted tweet has no media
+            return
+
+    else:
+        # no media, no quoted tweet
+        return
+
     for medium in media:
         if 'video_info' in medium:
             # FIXME: we're assuming one tweet only contains one video here

From 7db225a581a50de58581997cc0ccd8f9b1064dd2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 19 Jul 2020 02:17:22 +0200
Subject: [PATCH 0972/1225] [youtube] unescape html5player URLs for some videos

---
 src/you_get/extractors/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index f8630b8c1c..4f3a947e0f 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -439,6 +439,7 @@ def prepare(self, **kwargs):
         except:
             # VEVO
             if not self.html5player: return
+            self.html5player = self.html5player.replace('\/', '/') # unescape URL (for age-restricted videos)
             self.js = get_content(self.html5player)
 
             try:

From bf507e5e45e61223e898ab0d740fded90c35d69d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 19 Jul 2020 15:59:21 +0200
Subject: [PATCH 0973/1225] [processor.ffmpeg] separate output with '--'

- See https://github.com/soimort/you-get/commit/feffcb656ad2c33b17fb2e20598f8137fc69789c#r40158237
---
 src/you_get/processor/ffmpeg.py | 40 ++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index de78d5250b..11126c2730 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -60,7 +60,7 @@ def ffmpeg_concat_av(files, output, ext):
     for file in files:
         if os.path.isfile(file): params.extend(['-i', file])
     params.extend(['-c', 'copy'])
-    params.append(output)
+    params.extend(['--', output])
     if subprocess.call(params, stdin=STDIN):
         print('Merging without re-encode failed.\nTry again re-encoding audio... ', end="", flush=True)
         try: os.remove(output)
@@ -74,7 +74,7 @@ def ffmpeg_concat_av(files, output, ext):
             params.extend(['-strict', 'experimental'])
         elif ext == 'webm':
             params.extend(['-c:a', 'opus'])
-        params.append(output)
+        params.extend(['--', output])
         return subprocess.call(params, stdin=STDIN)
     else:
         return 0
@@ -83,7 +83,8 @@ def ffmpeg_convert_ts_to_mkv(files, output='output.mkv'):
     for file in files:
         if os.path.isfile(file):
             params = [FFMPEG] + LOGLEVEL
-            params.extend(['-y', '-i', file, output])
+            params.extend(['-y', '-i', file])
+            params.extend(['--', output])
             subprocess.call(params, stdin=STDIN)
 
     return
@@ -93,7 +94,8 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
         concat_list = generate_concat_list(files, output)
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
-                                        '-i', concat_list, '-c', 'copy', output]
+                                        '-i', concat_list, '-c', 'copy']
+        params.extend(['--', output])
         if subprocess.call(params, stdin=STDIN) == 0:
             os.remove(output + '.txt')
             return True
@@ -114,7 +116,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
     params.append(output + '.mpg')
     params += ['-vcodec', 'copy', '-acodec', 'copy']
-    params.append(output)
+    params.extend(['--', output])
 
     if subprocess.call(params, stdin=STDIN) == 0:
         for file in files:
@@ -131,7 +133,8 @@ def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
     for file in files:
         if os.path.isfile(file):
             params[-1] += file + '|'
-    params += ['-f', 'matroska', '-c', 'copy', output]
+    params += ['-f', 'matroska', '-c', 'copy']
+    params.extend(['--', output])
 
     try:
         if subprocess.call(params, stdin=STDIN) == 0:
@@ -148,7 +151,8 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
         concat_list = generate_concat_list(files, output)
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
                                         '-i', concat_list, '-c', 'copy',
-                                        '-bsf:a', 'aac_adtstoasc', output]
+                                        '-bsf:a', 'aac_adtstoasc']
+        params.extend(['--', output])
         subprocess.check_call(params, stdin=STDIN)
         os.remove(output + '.txt')
         return True
@@ -169,9 +173,10 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
         if os.path.isfile(f):
             params[-1] += f + '|'
     if FFMPEG == 'avconv':
-        params += ['-c', 'copy', output]
+        params += ['-c', 'copy']
     else:
-        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
+        params += ['-c', 'copy', '-absf', 'aac_adtstoasc']
+    params.extend(['--', output])
 
     if subprocess.call(params, stdin=STDIN) == 0:
         for file in files:
@@ -186,7 +191,8 @@ def ffmpeg_concat_mp3_to_mp3(files, output='output.mp3'):
     files = 'concat:' + '|'.join(files)
 
     params = [FFMPEG] + LOGLEVEL + ['-y']
-    params += ['-i', files, '-acodec', 'copy', output]
+    params += ['-i', files, '-acodec', 'copy']
+    params.extend(['--', output])
 
     subprocess.call(params)
 
@@ -199,7 +205,8 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         concat_list = generate_concat_list(files, output)
         params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
                                         '-i', concat_list, '-c', 'copy',
-                                        '-bsf:a', 'aac_adtstoasc', output]
+                                        '-bsf:a', 'aac_adtstoasc']
+        params.extend(['--', output])
         subprocess.check_call(params, stdin=STDIN)
         os.remove(output + '.txt')
         return True
@@ -220,9 +227,10 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
         if os.path.isfile(f):
             params[-1] += f + '|'
     if FFMPEG == 'avconv':
-        params += ['-c', 'copy', output]
+        params += ['-c', 'copy']
     else:
-        params += ['-c', 'copy', '-absf', 'aac_adtstoasc', output]
+        params += ['-c', 'copy', '-absf', 'aac_adtstoasc']
+    params.extend(['--', output])
 
     subprocess.check_call(params, stdin=STDIN)
     for file in files:
@@ -248,7 +256,7 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=
     ffmpeg_params.append(files)  #not the same here!!!!
 
     if FFMPEG == 'avconv':  #who cares?
-        ffmpeg_params += ['-c', 'copy', output]
+        ffmpeg_params += ['-c', 'copy']
     else:
         ffmpeg_params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
 
@@ -258,7 +266,7 @@ def ffmpeg_download_stream(files, title, ext, params={}, output_dir='.', stream=
                 ffmpeg_params.append(k)
                 ffmpeg_params.append(v)
 
-    ffmpeg_params.append(output)
+    ffmpeg_params.extend(['--', output])
 
     print(' '.join(ffmpeg_params))
 
@@ -286,7 +294,7 @@ def ffmpeg_concat_audio_and_video(files, output, ext):
         params.extend(['-c:v', 'copy'])
         params.extend(['-c:a', 'aac'])
         params.extend(['-strict', 'experimental'])
-        params.append(output+"."+ext)
+        params.extend(['--', output + "." + ext])
         return subprocess.call(params, stdin=STDIN)
     else:
         raise EnvironmentError('No ffmpeg found')

From 5da4245ed0c26262f2241202d886208e635296be Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 19 Jul 2020 16:17:28 +0200
Subject: [PATCH 0974/1225] version 0.4.1456

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index d5004187f2..e404e0c05a 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1432'
+__version__ = '0.4.1456'

From 80d88a1331720436bdfa85806709481d48368870 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 26 Jul 2020 15:06:03 +0200
Subject: [PATCH 0975/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 28 +++++++++++++++++++---------
 tests/test.py                    | 15 +++++++++++----
 2 files changed, 30 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 632e44c2d5..c2a0eb8dcf 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -6,15 +6,25 @@
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url, faker=True)
-    title = r1(r'<title.*?>(.*?)</title>', html)
-    video_id = r1(r'/video/(\d+)', url) or r1(r'musical\?id=(\d+)', html)
-    title = '%s [%s]' % (title, video_id)
-    source = r1(r'<video .*?src="([^"]+)"', html) or r1(r'"contentUrl":"([^"]+)"', html)
-    mime, ext, size = url_info(source)
-
-    print_info(site_info, title, mime, size)
-    if not info_only:
-        download_urls([source], title, ext, size, output_dir, merge=merge)
+
+    data = r1(r'<script id="__NEXT_DATA__".*?>(.*?)</script>', html)
+    info = json.loads(data)
+    videoData = info['props']['pageProps']['videoData']
+    urls = videoData['itemInfos']['video']['urls']
+    videoId = videoData['itemInfos']['id']
+    uniqueId = videoData['authorInfos'].get('uniqueId')
+    nickName = videoData['authorInfos'].get('nickName')
+
+    for i, url in enumerate(urls):
+        title = '%s [%s]' % (nickName or uniqueId, videoId)
+        if len(urls) > 1:
+            title = '%s [%s]' % (title, i)
+
+        mime, ext, size = url_info(url)
+
+        print_info(site_info, title, mime, size)
+        if not info_only:
+            download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
 
 site_info = "TikTok.com"
 download = tiktok_download
diff --git a/tests/test.py b/tests/test.py
index 38db15ddc4..00bd4cbb7e 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -9,7 +9,8 @@
     missevan,
     acfun,
     bilibili,
-    soundcloud
+    soundcloud,
+    tiktok
 )
 
 
@@ -53,9 +54,15 @@ def test_soundcloud(self):
             'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
         )
         ## playlist
-        soundcloud.download(
-            'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
-        )
+        #soundcloud.download(
+        #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
+        #)
+
+    def tests_tiktok(self):
+        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
+        tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True)
+
 
 if __name__ == '__main__':
     unittest.main()

From c14605f7011ebe40c01c10e3063a4c25c88a4f04 Mon Sep 17 00:00:00 2001
From: MMMartt <nielumartin@hotmail.com>
Date: Tue, 18 Aug 2020 17:05:58 +0800
Subject: [PATCH 0976/1225] fix acfun download fail

---
 src/you_get/extractors/acfun.py | 39 ++++++++++++++-------------------
 tests/test.py                   |  1 +
 2 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index b83c28594d..9205b1b84e 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -111,6 +111,18 @@ def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=Fals
 def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
 
+    def getM3u8UrlFromCurrentVideoInfo(currentVideoInfo):
+        if 'playInfos' in currentVideoInfo:
+            return currentVideoInfo['playInfos'][0]['playUrls'][0]
+        elif 'ksPlayJson' in currentVideoInfo:
+            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
+            representation = ksPlayJson.get('adaptationSet')[0].get('representation')
+            reps = []
+            for one in representation:
+                reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
+            return max(reps)[1]
+
+
     if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
         html = get_content(url, headers=fake_headers)
         json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
@@ -122,37 +134,18 @@ def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         if len(video_list) > 1:
             title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
         currentVideoInfo = json_data.get('currentVideoInfo')
-        if 'playInfos' in currentVideoInfo:
-            m3u8_url = currentVideoInfo['playInfos'][0]['playUrls'][0]
-        elif 'ksPlayJson' in currentVideoInfo:
-            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
-            representation = ksPlayJson.get('adaptationSet').get('representation')
-            reps = []
-            for one in representation:
-                reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
-            m3u8_url = max(reps)[1]
-
+        m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
     elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", url):
         html = get_content(url, headers=fake_headers)
-        tag_script = match1(html, r'<script>window\.pageInfo([^<]+)</script>')
+        tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
         json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
         json_data = json.loads(json_text)
         title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
         vid = str(json_data['videoId'])
         up = "acfun"
 
-        play_info = get_content("https://www.acfun.cn/rest/pc-direct/play/playInfo/m3u8Auto?videoId=" + vid, headers=fake_headers)
-        play_url = json.loads(play_info)['playInfo']['streams'][0]['playUrls'][0]
-        m3u8_all_qualities_file = get_content(play_url)
-        m3u8_all_qualities_lines = m3u8_all_qualities_file.split('#EXT-X-STREAM-INF:')[1:]
-        highest_quality_line = m3u8_all_qualities_lines[0]
-        for line in m3u8_all_qualities_lines:
-            bandwith = int(match1(line, r'BANDWIDTH=(\d+)'))
-            if bandwith > int(match1(highest_quality_line, r'BANDWIDTH=(\d+)')):
-                highest_quality_line = line
-        #TODO: 应由用户指定清晰度
-        m3u8_url = match1(highest_quality_line, r'\n([^#\n]+)$')
-        m3u8_url = play_url[:play_url.rfind("/")+1] + m3u8_url
+        currentVideoInfo = json_data.get('currentVideoInfo')
+        m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
 
     else:
         raise NotImplemented
diff --git a/tests/test.py b/tests/test.py
index 00bd4cbb7e..33503b8de0 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -39,6 +39,7 @@ def test_youtube(self):
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
+        acfun.download('https://www.acfun.cn/bangumi/aa6002986', info_only=True)
 
     def test_bilibil(self):
         bilibili.download(

From 5adb4caa738464a0730cf70fee34ae8191f100b5 Mon Sep 17 00:00:00 2001
From: Shen <960821@gmail.com>
Date: Mon, 7 Sep 2020 14:29:00 +0800
Subject: [PATCH 0977/1225] fix bilibili 4k

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 94e5479f65..7ea626f89d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -62,7 +62,7 @@ def bilibili_headers(referer=None, cookie=None):
 
     @staticmethod
     def bilibili_api(avid, cid, qn=0):
-        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16' % (avid, cid, qn)
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16&fourk=1' % (avid, cid, qn)
 
     @staticmethod
     def bilibili_audio_api(sid):

From 398068e13e8b7052903f89d1caf30023e8639e78 Mon Sep 17 00:00:00 2001
From: jseagull <jseagull@aliyun.com>
Date: Mon, 14 Sep 2020 15:28:29 +0800
Subject: [PATCH 0978/1225] fix iqiyi playlist extrator

---
 src/you_get/extractors/iqiyi.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 3fe932091e..2a48791cc9 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -119,10 +119,10 @@ def download_playlist_by_url(self, url, **kwargs):
         self.url = url
 
         video_page = get_content(url)
-        videos = set(re.findall(r'<a href="(http://www\.iqiyi\.com/v_[^"]+)"', video_page))
+        videos = set(re.findall(r'<a href="(?=https?:)?(//www\.iqiyi\.com/v_[^"]+)"', video_page))
 
         for video in videos:
-            self.__class__().download_by_url(video, **kwargs)
+            self.__class__().download_by_url('https:' + video, **kwargs)
 
     def prepare(self, **kwargs):
         assert self.url or self.vid
@@ -153,7 +153,7 @@ def prepare(self, **kwargs):
             except Exception as e:
                 log.i("vd: {} is not handled".format(stream['vd']))
                 log.i("info is {}".format(stream))
-    
+
 
     def download(self, **kwargs):
         """Override the original one
@@ -201,7 +201,7 @@ def download(self, **kwargs):
             if not urls:
                 log.wtf('[Failed] Cannot extract video source.')
             # For legacy main()
-            
+
             #Here's the change!!
             download_url_ffmpeg(urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], stream=False)
 
@@ -215,7 +215,7 @@ def download(self, **kwargs):
                 with open(os.path.join(kwargs['output_dir'], filename),
                           'w', encoding='utf-8') as x:
                     x.write(srt)
-                print('Done.')    
+                print('Done.')
 
 '''
         if info["code"] != "A000000":

From 8a675aec761f5c7b3c27084f62ad7a0c851310a2 Mon Sep 17 00:00:00 2001
From: Doug Benham <dougbenham@cox.net>
Date: Fri, 18 Sep 2020 14:25:38 -0700
Subject: [PATCH 0979/1225] Fixed tiktok extraction

---
 src/you_get/extractors/tiktok.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index c2a0eb8dcf..4843ced8c2 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -15,16 +15,16 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     uniqueId = videoData['authorInfos'].get('uniqueId')
     nickName = videoData['authorInfos'].get('nickName')
 
-    for i, url in enumerate(urls):
+    for i, videoUrl in enumerate(urls):
         title = '%s [%s]' % (nickName or uniqueId, videoId)
         if len(urls) > 1:
             title = '%s [%s]' % (title, i)
 
-        mime, ext, size = url_info(url)
+        mime, ext, size = url_info(videoUrl, headers={'Referer': url})
 
         print_info(site_info, title, mime, size)
         if not info_only:
-            download_urls([url], title, ext, size, output_dir=output_dir, merge=merge)
+            download_urls([videoUrl], title, ext, size, output_dir=output_dir, merge=merge, headers={'Referer': url})
 
 site_info = "TikTok.com"
 download = tiktok_download

From 4abc4bd7cfa1ec787a59a91480df28e458a12fe0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 19 Sep 2020 22:57:06 +0200
Subject: [PATCH 0980/1225] [instagram] fix vid extraction for Reels

---
 src/you_get/extractors/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 567e0dd7d7..86905a77b2 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -8,7 +8,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     url = r1(r'([^?]*)', url)
     html = get_html(url)
 
-    vid = r1(r'instagram.com/p/([^/]+)', url)
+    vid = r1(r'instagram.com/\w+/([^/]+)', url)
     description = r1(r'<meta property="og:title" content="([^"]*)"', html)
     title = "{} [{}]".format(description.replace("\n", " "), vid)
     stream = r1(r'<meta property="og:video" content="([^"]*)"', html)

From da1d772cd84326b2258c3b8b889324105317573f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 20 Sep 2020 17:05:47 +0200
Subject: [PATCH 0981/1225] [imgur] fix

---
 src/you_get/extractors/imgur.py | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index cc5dc9fd88..519fa245ca 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -52,20 +52,16 @@ def prepare(self, **kwargs):
         else:
             # gallery image
             content = get_content(self.url)
-            image = json.loads(match1(content, r'image\s*:\s*({.*}),'))
-            ext = image['ext']
+            url = match1(content, r'(https?://i.imgur.com/[^"]+)')
+            _, container, size = url_info(url)
             self.streams = {
                 'original': {
-                    'src': ['http://i.imgur.com/%s%s' % (image['hash'], ext)],
-                    'size': image['size'],
-                    'container': ext[1:]
-                },
-                'thumbnail': {
-                    'src': ['http://i.imgur.com/%ss%s' % (image['hash'], '.jpg')],
-                    'container': 'jpg'
+                    'src': [url],
+                    'size': size,
+                    'container': container
                 }
             }
-            self.title = image['title'] or image['hash']
+            self.title = r1(r'i\.imgur\.com/([^./]*)', url)
 
     def extract(self, **kwargs):
         if 'stream_id' in kwargs and kwargs['stream_id']:

From 00e2ce3f48b0cc3a8bcc8fe07cdc1892783b74e1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 20 Sep 2020 17:14:34 +0200
Subject: [PATCH 0982/1225] [test] remove case for acfun bangumi that is
 inaccessible from non-China IP

---
 tests/test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index 33503b8de0..00bd4cbb7e 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -39,7 +39,6 @@ def test_youtube(self):
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
-        acfun.download('https://www.acfun.cn/bangumi/aa6002986', info_only=True)
 
     def test_bilibil(self):
         bilibili.download(

From 5c9ec6c4f300dda24fcf0ad6b75a45e3cec46536 Mon Sep 17 00:00:00 2001
From: johnsmith2077 <johnsmith2077@foxmail.com>
Date: Sun, 4 Oct 2020 05:12:56 +0800
Subject: [PATCH 0983/1225] add format selection for AcFun

---
 src/you_get/extractors/acfun.py | 361 ++++++++++++++++++--------------
 1 file changed, 203 insertions(+), 158 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 9205b1b84e..cd27592789 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -1,168 +1,213 @@
 #!/usr/bin/env python
 
-__all__ = ['acfun_download']
-
 from ..common import *
+from ..extractor import VideoExtractor
+
+class AcFun(VideoExtractor):
+    name = "AcFun"
+
+    stream_types = [
+        {'id': '2160P', 'qualityType': '2160p'},
+        {'id': '1080P60', 'qualityType': '1080p60'},
+        {'id': '720P60', 'qualityType': '720p60'},
+        {'id': '1080P+', 'qualityType': '1080p+'},
+        {'id': '1080P', 'qualityType': '1080p'},
+        {'id': '720P', 'qualityType': '720p'},
+        {'id': '540P', 'qualityType': '540p'},
+        {'id': '360P', 'qualityType': '360p'}
+    ]    
+
+    def prepare(self, **kwargs):
+        assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', self.url)
+
+        if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', self.url):
+            html = get_content(self.url, headers=fake_headers)
+            json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
+            json_data = json.loads(json_text)
+            vid = json_data.get('currentVideoInfo').get('id')
+            up = json_data.get('user').get('name')
+            self.title = json_data.get('title')
+            video_list = json_data.get('videoList')
+            if len(video_list) > 1:
+                self.title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
+            currentVideoInfo = json_data.get('currentVideoInfo')
+
+        elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", self.url):
+            html = get_content(self.url, headers=fake_headers)
+            tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
+            json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
+            json_data = json.loads(json_text)
+            self.title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
+            vid = str(json_data['videoId'])
+            up = "acfun"
+            currentVideoInfo = json_data.get('currentVideoInfo')
 
-from .le import letvcloud_download_by_vu
-from .qq import qq_download_by_vid
-from .sina import sina_download_by_vid
-from .tudou import tudou_download_by_iid
-from .youku import youku_download_by_vid
-
-import json
-import re
-import base64
-import time
-
-def get_srt_json(id):
-    url = 'http://danmu.aixifan.com/V2/%s' % id
-    return get_content(url)
-
-def youku_acfun_proxy(vid, sign, ref):
-    endpoint = 'http://player.acfun.cn/flash_data?vid={}&ct=85&ev=3&sign={}&time={}'
-    url = endpoint.format(vid, sign, str(int(time.time() * 1000)))
-    json_data = json.loads(get_content(url, headers=dict(referer=ref)))['data']
-    enc_text = base64.b64decode(json_data)
-    dec_text = rc4(b'8bdc7e1a', enc_text).decode('utf8')
-    youku_json = json.loads(dec_text)
-
-    yk_streams = {}
-    for stream in youku_json['stream']:
-        tp = stream['stream_type']
-        yk_streams[tp] = [], stream['total_size']
-        if stream.get('segs'):
-            for seg in stream['segs']:
-                yk_streams[tp][0].append(seg['url'])
-        else:
-            yk_streams[tp] = stream['m3u8'], stream['total_size']
-
-    return yk_streams
-
-def acfun_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False, **kwargs):
-    """str, str, str, bool, bool ->None
-
-    Download Acfun video by vid.
-
-    Call Acfun API, decide which site to use, and pass the job to its
-    extractor.
-    """
-
-    #first call the main parasing API
-    info = json.loads(get_content('http://www.acfun.cn/video/getVideo.aspx?id=' + vid, headers=fake_headers))
-
-    sourceType = info['sourceType']
-
-    #decide sourceId to know which extractor to use
-    if 'sourceId' in info: sourceId = info['sourceId']
-    # danmakuId = info['danmakuId']
-
-    #call extractor decided by sourceId
-    if sourceType == 'sina':
-        sina_download_by_vid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif sourceType == 'youku':
-        youku_download_by_vid(sourceId, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
-    elif sourceType == 'tudou':
-        tudou_download_by_iid(sourceId, title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif sourceType == 'qq':
-        qq_download_by_vid(sourceId, title, True, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif sourceType == 'letv':
-        letvcloud_download_by_vu(sourceId, '2d8c027396', title, output_dir=output_dir, merge=merge, info_only=info_only)
-    elif sourceType == 'zhuzhan':
-        #As in Jul.28.2016, Acfun is using embsig to anti hotlink so we need to pass this
-#In Mar. 2017 there is a dedicated ``acfun_proxy'' in youku cloud player
-#old code removed
-        url = 'http://www.acfun.cn/v/ac' + vid
-        yk_streams = youku_acfun_proxy(info['sourceId'], info['encode'], url)
-        seq = ['mp4hd3', 'mp4hd2', 'mp4hd', 'flvhd']
-        for t in seq:
-            if yk_streams.get(t):
-                preferred = yk_streams[t]
-                break
-#total_size in the json could be incorrect(F.I. 0)
-        size = 0
-        for url in preferred[0]:
-            _, _, seg_size = url_info(url)
-            size += seg_size
-#fallback to flvhd is not quite possible
-        if re.search(r'fid=[0-9A-Z\-]*.flv', preferred[0][0]):
-            ext = 'flv'
         else:
-            ext = 'mp4'
-        print_info(site_info, title, ext, size)
-        if not info_only:
-            download_urls(preferred[0], title, ext, size, output_dir=output_dir, merge=merge)
-    else:
-        raise NotImplementedError(sourceType)
-
-    if not info_only and not dry_run:
-        if not kwargs['caption']:
-            print('Skipping danmaku.')
-            return
-        try:
-            title = get_filename(title)
-            print('Downloading %s ...\n' % (title + '.cmt.json'))
-            cmt = get_srt_json(vid)
-            with open(os.path.join(output_dir, title + '.cmt.json'), 'w', encoding='utf-8') as x:
-                x.write(cmt)
-        except:
-            pass
-
-def acfun_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
-
-    def getM3u8UrlFromCurrentVideoInfo(currentVideoInfo):
-        if 'playInfos' in currentVideoInfo:
-            return currentVideoInfo['playInfos'][0]['playUrls'][0]
-        elif 'ksPlayJson' in currentVideoInfo:
-            ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
+            raise NotImplemented            
+
+        if 'ksPlayJson' in currentVideoInfo:
+            durationMillis = currentVideoInfo['durationMillis']
+            ksPlayJson = ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
             representation = ksPlayJson.get('adaptationSet')[0].get('representation')
-            reps = []
-            for one in representation:
-                reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
-            return max(reps)[1]
-
-
-    if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
-        html = get_content(url, headers=fake_headers)
-        json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
-        json_data = json.loads(json_text)
-        vid = json_data.get('currentVideoInfo').get('id')
-        up = json_data.get('user').get('name')
-        title = json_data.get('title')
-        video_list = json_data.get('videoList')
-        if len(video_list) > 1:
-            title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
-        currentVideoInfo = json_data.get('currentVideoInfo')
-        m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
-    elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", url):
-        html = get_content(url, headers=fake_headers)
-        tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
-        json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
-        json_data = json.loads(json_text)
-        title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
-        vid = str(json_data['videoId'])
-        up = "acfun"
-
-        currentVideoInfo = json_data.get('currentVideoInfo')
-        m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
-
-    else:
-        raise NotImplemented
-
-    assert title and m3u8_url
-    title = unescape_html(title)
-    title = escape_file_path(title)
-    p_title = r1('active">([^<]+)', html)
-    title = '%s (%s)' % (title, up)
-    if p_title:
-        title = '%s - %s' % (title, p_title)
-
-    print_info(site_info, title, 'm3u8', float('inf'))
-    if not info_only:
-        download_url_ffmpeg(m3u8_url, title, 'mp4', output_dir=output_dir, merge=merge)
+            stream_list = representation
+
+        for stream in stream_list:
+            m3u8_url = stream["url"]
+            size = durationMillis * stream["avgBitrate"] / 8
+            # size = float('inf')
+            container = 'mp4'
+            stream_id = stream["qualityLabel"]
+            quality = stream["qualityType"]
+            
+            stream_data = dict(src=m3u8_url, size=size, container=container, quality=quality)
+            self.streams[stream_id] = stream_data
+
+        assert self.title and m3u8_url
+        self.title = unescape_html(self.title)
+        self.title = escape_file_path(self.title)
+        p_title = r1('active">([^<]+)', html)
+        self.title = '%s (%s)' % (self.title, up)
+        if p_title:
+            self.title = '%s - %s' % (self.title, p_title)       
+
+
+    def download(self, **kwargs):
+        if 'json_output' in kwargs and kwargs['json_output']:
+            json_output.output(self)
+        elif 'info_only' in kwargs and kwargs['info_only']:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Display the stream
+                stream_id = kwargs['stream_id']
+                if 'index' not in kwargs:
+                    self.p(stream_id)
+                else:
+                    self.p_i(stream_id)
+            else:
+                # Display all available streams
+                if 'index' not in kwargs:
+                    self.p([])
+                else:
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    self.p_i(stream_id)
+
+        else:
+            if 'stream_id' in kwargs and kwargs['stream_id']:
+                # Download the stream
+                stream_id = kwargs['stream_id']
+            else:
+                stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+
+            if 'index' not in kwargs:
+                self.p(stream_id)
+            else:
+                self.p_i(stream_id)
+            if stream_id in self.streams:
+                url = self.streams[stream_id]['src']
+                ext = self.streams[stream_id]['container']
+                total_size = self.streams[stream_id]['size']
+
+
+            if ext == 'm3u8' or ext == 'm4a':
+                ext = 'mp4'
+
+            if not url:
+                log.wtf('[Failed] Cannot extract video source.')
+            # For legacy main()
+            headers = {}
+            if self.ua is not None:
+                headers['User-Agent'] = self.ua
+            if self.referer is not None:
+                headers['Referer'] = self.referer
+
+            download_url_ffmpeg(url, self.title, ext, output_dir=kwargs['output_dir'], merge=kwargs['merge'])                           
+
+            if 'caption' not in kwargs or not kwargs['caption']:
+                print('Skipping captions or danmaku.')
+                return
+
+            for lang in self.caption_tracks:
+                filename = '%s.%s.srt' % (get_filename(self.title), lang)
+                print('Saving %s ... ' % filename, end="", flush=True)
+                srt = self.caption_tracks[lang]
+                with open(os.path.join(kwargs['output_dir'], filename),
+                          'w', encoding='utf-8') as x:
+                    x.write(srt)
+                print('Done.')
+
+            if self.danmaku is not None and not dry_run:
+                filename = '{}.cmt.xml'.format(get_filename(self.title))
+                print('Downloading {} ...\n'.format(filename))
+                with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
+                    fp.write(self.danmaku)
+
+            if self.lyrics is not None and not dry_run:
+                filename = '{}.lrc'.format(get_filename(self.title))
+                print('Downloading {} ...\n'.format(filename))
+                with open(os.path.join(kwargs['output_dir'], filename), 'w', encoding='utf8') as fp:
+                    fp.write(self.lyrics)
+
+            # For main_dev()
+            #download_urls(urls, self.title, self.streams[stream_id]['container'], self.streams[stream_id]['size'])
+        keep_obj = kwargs.get('keep_obj', False)
+        if not keep_obj:
+            self.__init__()
+
+
+    def acfun_download(self, url, output_dir='.', merge=True, info_only=False, **kwargs):
+        assert re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/(\D|bangumi)/\D\D(\d+)', url)
+
+        def getM3u8UrlFromCurrentVideoInfo(currentVideoInfo):
+            if 'playInfos' in currentVideoInfo:
+                return currentVideoInfo['playInfos'][0]['playUrls'][0]
+            elif 'ksPlayJson' in currentVideoInfo:
+                ksPlayJson = json.loads( currentVideoInfo['ksPlayJson'] )
+                representation = ksPlayJson.get('adaptationSet')[0].get('representation')
+                reps = []
+                for one in representation:
+                    reps.append( (one['width']* one['height'], one['url'], one['backupUrl']) )
+                return max(reps)[1]
+
+
+        if re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/\D/\D\D(\d+)', url):
+            html = get_content(url, headers=fake_headers)
+            json_text = match1(html, r"(?s)videoInfo\s*=\s*(\{.*?\});")
+            json_data = json.loads(json_text)
+            vid = json_data.get('currentVideoInfo').get('id')
+            up = json_data.get('user').get('name')
+            title = json_data.get('title')
+            video_list = json_data.get('videoList')
+            if len(video_list) > 1:
+                title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
+            currentVideoInfo = json_data.get('currentVideoInfo')
+            m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
+        elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", url):
+            html = get_content(url, headers=fake_headers)
+            tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
+            json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
+            json_data = json.loads(json_text)
+            title = json_data['bangumiTitle'] + " " + json_data['episodeName'] + " " + json_data['title']
+            vid = str(json_data['videoId'])
+            up = "acfun"
+
+            currentVideoInfo = json_data.get('currentVideoInfo')
+            m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
+
+        else:
+            raise NotImplemented
 
+        assert title and m3u8_url
+        title = unescape_html(title)
+        title = escape_file_path(title)
+        p_title = r1('active">([^<]+)', html)
+        title = '%s (%s)' % (title, up)
+        if p_title:
+            title = '%s - %s' % (title, p_title)
+
+        print_info(site_info, title, 'm3u8', float('inf'))
+        if not info_only:
+            download_url_ffmpeg(m3u8_url, title, 'mp4', output_dir=output_dir, merge=merge)
 
+site = AcFun()
 site_info = "AcFun.cn"
-download = acfun_download
+download = site.download_by_url
 download_playlist = playlist_not_supported('acfun')

From faff58a148f323064fcaef3745a9be95e5066bd0 Mon Sep 17 00:00:00 2001
From: cmsxbc <i@cmsis.me>
Date: Sun, 11 Oct 2020 23:40:00 +0800
Subject: [PATCH 0984/1225] fix bilibili space videos

---
 src/you_get/extractors/bilibili.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 7ea626f89d..cdcccf2063 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -114,7 +114,7 @@ def bilibili_space_favlist_api(fid, pn=1, ps=20):
 
     @staticmethod
     def bilibili_space_video_api(mid, pn=1, ps=100):
-        return 'https://space.bilibili.com/ajax/member/getSubmitVideos?mid=%s&page=%s&pagesize=%s&order=0&jsonp=jsonp' % (mid, pn, ps)
+        return "https://api.bilibili.com/x/space/arc/search?mid=%s&pn=%s&ps=%s&tid=0&keyword=&order=pubdate&jsonp=jsonp" % (mid, pn, ps)
 
     @staticmethod
     def bilibili_vc_api(video_id):
@@ -734,15 +734,15 @@ def download_playlist_by_url(self, url, **kwargs):
             api_url = self.bilibili_space_video_api(mid)
             api_content = get_content(api_url, headers=self.bilibili_headers())
             videos_info = json.loads(api_content)
-            pc = videos_info['data']['pages']
+            pc = videos_info['data']['page']['count'] // videos_info['data']['page']['ps']
 
             for pn in range(1, pc + 1):
                 api_url = self.bilibili_space_video_api(mid, pn=pn)
                 api_content = get_content(api_url, headers=self.bilibili_headers())
                 videos_info = json.loads(api_content)
 
-                epn, i = len(videos_info['data']['vlist']), 0
-                for video in videos_info['data']['vlist']:
+                epn, i = len(videos_info['data']['list']['vlist']), 0
+                for video in videos_info['data']['list']['vlist']:
                     i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                     url = 'https://www.bilibili.com/video/av%s' % video['aid']
                     self.__class__().download_playlist_by_url(url, **kwargs)

From 205470ec116654608ddd97390bd885ba6df100b1 Mon Sep 17 00:00:00 2001
From: zhouyuan1 <zhouyuan1@staff.weibo.com>
Date: Mon, 12 Oct 2020 13:22:17 +0800
Subject: [PATCH 0985/1225] add support for socks5 proxy using username and
 password config

---
 src/you_get/common.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 9c56b5c2c0..515214079e 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1422,12 +1422,27 @@ def load_cookies(cookiefile):
 def set_socks_proxy(proxy):
     try:
         import socks
-        socks_proxy_addrs = proxy.split(':')
-        socks.set_default_proxy(
-            socks.SOCKS5,
-            socks_proxy_addrs[0],
-            int(socks_proxy_addrs[1])
-        )
+        if '@' in proxy:
+            proxy_info = proxy.split("@")
+            socks_proxy_addrs = proxy_info[1].split(':')
+            socks_proxy_auth = proxy_info[0].split(":")
+            print(socks_proxy_auth[0]+" "+socks_proxy_auth[1]+" "+socks_proxy_addrs[0]+" "+socks_proxy_addrs[1])
+            socks.set_default_proxy(
+                socks.SOCKS5,
+                socks_proxy_addrs[0],
+                int(socks_proxy_addrs[1]),
+                True,
+                socks_proxy_auth[0],
+                socks_proxy_auth[1]
+            )
+        else:
+           socks_proxy_addrs = proxy.split(':')
+           print(socks_proxy_addrs[0]+" "+socks_proxy_addrs[1])
+           socks.set_default_proxy(
+               socks.SOCKS5,
+               socks_proxy_addrs[0],
+               int(socks_proxy_addrs[1]),
+           )
         socket.socket = socks.socksocket
 
         def getaddrinfo(*args):
@@ -1565,7 +1580,7 @@ def print_version():
         '--no-proxy', action='store_true', help='Never use a proxy'
     )
     proxy_grp.add_argument(
-        '-s', '--socks-proxy', metavar='HOST:PORT',
+        '-s', '--socks-proxy', metavar='HOST:PORT or USERNAME:PASSWORD@HOST:PORT',
         help='Use an SOCKS5 proxy for downloading'
     )
 

From c271363585021c9ee86e9c6e08e35d48d9fbf159 Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Wed, 14 Oct 2020 23:11:09 +0800
Subject: [PATCH 0986/1225] Correct a typo in iqiyi.py

---
 src/you_get/extractors/iqiyi.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 2a48791cc9..d138a49fb8 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -20,7 +20,7 @@
    use @fffonion 's method in #617.
    Add trace AVM(asasm) code in Iqiyi's encode function where the salt is put into the encode array and reassemble by RABCDasm(or WinRABCDasm),then use Fiddler to response modified file to replace the src file with its AutoResponder function ,set browser Fiddler proxy and play with !debug version! Flash Player ,finially get result in flashlog.txt(its location can be easily found in search engine).
    Code Like (without letters after #comment:),it just do the job : trace("{IQIYI_SALT}:"+salt_array.join(""))
-   ```(Postion After getTimer)
+   ```(Position After getTimer)
      findpropstrict      QName(PackageNamespace(""), "trace")
      pushstring          "{IQIYI_SALT}:" #comment for you to locate the salt
      getscopeobject      1

From 5d59f76a41bca9a88495d2b721dc5618a1a568ba Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 25 Oct 2020 17:02:28 +0100
Subject: [PATCH 0987/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 50 +++++++++++++++++++++-----------
 1 file changed, 33 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 4843ced8c2..2ef0522666 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -5,26 +5,42 @@
 from ..common import *
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_html(url, faker=True)
+    referUrl = url.split('?')[0]
+    headers = fake_headers
 
+    # trick or treat
+    html = get_content(url, headers=headers)
     data = r1(r'<script id="__NEXT_DATA__".*?>(.*?)</script>', html)
     info = json.loads(data)
-    videoData = info['props']['pageProps']['videoData']
-    urls = videoData['itemInfos']['video']['urls']
-    videoId = videoData['itemInfos']['id']
-    uniqueId = videoData['authorInfos'].get('uniqueId')
-    nickName = videoData['authorInfos'].get('nickName')
-
-    for i, videoUrl in enumerate(urls):
-        title = '%s [%s]' % (nickName or uniqueId, videoId)
-        if len(urls) > 1:
-            title = '%s [%s]' % (title, i)
-
-        mime, ext, size = url_info(videoUrl, headers={'Referer': url})
-
-        print_info(site_info, title, mime, size)
-        if not info_only:
-            download_urls([videoUrl], title, ext, size, output_dir=output_dir, merge=merge, headers={'Referer': url})
+    wid = info['props']['initialProps']['$wid']
+    cookie = 'tt_webid=%s; tt_webid_v2=%s' % (wid, wid)
+
+    # here's the cookie
+    headers['Cookie'] = cookie
+
+    # try again
+    html = get_content(url, headers=headers)
+    data = r1(r'<script id="__NEXT_DATA__".*?>(.*?)</script>', html)
+    info = json.loads(data)
+    wid = info['props']['initialProps']['$wid']
+    cookie = 'tt_webid=%s; tt_webid_v2=%s' % (wid, wid)
+
+    videoData = info['props']['pageProps']['itemInfo']['itemStruct']
+    videoId = videoData['id']
+    videoUrl = videoData['video']['downloadAddr']
+    uniqueId = videoData['author'].get('uniqueId')
+    nickName = videoData['author'].get('nickname')
+
+    title = '%s [%s]' % (nickName or uniqueId, videoId)
+
+    # we also need the referer
+    headers['Referer'] = referUrl
+
+    mime, ext, size = url_info(videoUrl, headers=headers)
+
+    print_info(site_info, title, mime, size)
+    if not info_only:
+        download_urls([videoUrl], title, ext, size, output_dir=output_dir, merge=merge, headers=headers)
 
 site_info = "TikTok.com"
 download = tiktok_download

From 517b8c090b719702518d4ccf5d4f217b9251adcd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 27 Oct 2020 17:20:28 +0100
Subject: [PATCH 0988/1225] [youtube] fix playlist extraction

---
 src/you_get/extractors/youtube.py | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 4f3a947e0f..e6388e985a 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -157,34 +157,22 @@ def download_playlist_by_url(self, url, **kwargs):
             log.wtf('[Failed] Unsupported URL pattern.')
 
         video_page = get_content('https://www.youtube.com/playlist?list=%s' % playlist_id)
-        from html.parser import HTMLParser
-        videos = sorted([HTMLParser().unescape(video)
-                         for video in re.findall(r'<a href="(/watch\?[^"]+)"', video_page)
-                         if parse_query_param(video, 'index')],
-                        key=lambda video: parse_query_param(video, 'index'))
-
-        # Parse browse_ajax page for more videos to load
-        load_more_href = match1(video_page, r'data-uix-load-more-href="([^"]+)"')
-        while load_more_href:
-            browse_ajax = get_content('https://www.youtube.com/%s' % load_more_href)
-            browse_data = json.loads(browse_ajax)
-            load_more_widget_html = browse_data['load_more_widget_html']
-            content_html = browse_data['content_html']
-            vs = set(re.findall(r'href="(/watch\?[^"]+)"', content_html))
-            videos += sorted([HTMLParser().unescape(video)
-                              for video in list(vs)
-                              if parse_query_param(video, 'index')])
-            load_more_href = match1(load_more_widget_html, r'data-uix-load-more-href="([^"]+)"')
+        ytInitialData = json.loads(match1(video_page, r'window\["ytInitialData"\]\s*=\s*(.+);'))
+
+        tab0 = ytInitialData['contents']['twoColumnBrowseResultsRenderer']['tabs'][0]
+        itemSection0 = tab0['tabRenderer']['content']['sectionListRenderer']['contents'][0]
+        playlistVideoList0 = itemSection0['itemSectionRenderer']['contents'][0]
+        videos = playlistVideoList0['playlistVideoListRenderer']['contents']
 
         self.title = re.search(r'<meta name="title" content="([^"]+)"', video_page).group(1)
         self.p_playlist()
-        for video in videos:
-            vid = parse_query_param(video, 'v')
-            index = parse_query_param(video, 'index')
+        for index, video in enumerate(videos, 1):
+            vid = video['playlistVideoRenderer']['videoId']
             try:
                 self.__class__().download_by_url(self.__class__.get_url_from_vid(vid), index=index, **kwargs)
             except:
                 pass
+        # FIXME: show DASH stream sizes (by default) for playlist videos
 
     def prepare(self, **kwargs):
         assert self.url or self.vid

From cc4f211382287b28b6434dafdc69a3607eb3d9d8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 27 Oct 2020 18:19:28 +0100
Subject: [PATCH 0989/1225] version 0.4.1475

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e404e0c05a..b469651960 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1456'
+__version__ = '0.4.1475'

From d0225b4f469c896363bdb96bbdd7a70675a9f0d1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 28 Oct 2020 02:12:07 +0100
Subject: [PATCH 0990/1225] [youtube] fix videos whose page ytplayer_config
 does not contain assets

---
 src/you_get/extractors/youtube.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index e6388e985a..19a720819c 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -204,13 +204,22 @@ def prepare(self, **kwargs):
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 try:
                     ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-                    self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
+
                     # Workaround: get_video_info returns bad s. Why?
                     if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
                         stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
                     else:
                         stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
                     #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
+
+                    if 'assets' in ytplayer_config:
+                        self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
+                    elif re.search('([^"]*/base\.js)"', video_page):
+                        self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                        self.html5player = self.html5player.replace('\/', '/') # unescape URL
+                    else:
+                        self.html5player = None
+
                 except:
                     if 'url_encoded_fmt_stream_map' not in video_info:
                         stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']

From 0e004039ffd452b5635c42e616a8e38adea5de98 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 28 Oct 2020 02:14:05 +0100
Subject: [PATCH 0991/1225] [youtube] decipher -> s_to_sig

---
 src/you_get/extractors/youtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 19a720819c..ac62e57c3a 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -68,7 +68,7 @@ class YouTube(VideoExtractor):
          'audio_encoding': 'AAC', 'audio_bitrate': '24'},
     ]
 
-    def decipher(js, s):
+    def s_to_sig(js, s):
         # Examples:
         # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
         # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
@@ -490,13 +490,13 @@ def prepare(self, **kwargs):
                 if stream['type'].startswith('audio/mp4'):
                     dash_mp4_a_url = stream['url']
                     if 's' in stream:
-                        sig = self.__class__.decipher(self.js, stream['s'])
+                        sig = self.__class__.s_to_sig(self.js, stream['s'])
                         dash_mp4_a_url += '&sig={}'.format(sig)
                     dash_mp4_a_size = stream['clen']
                 elif stream['type'].startswith('audio/webm'):
                     dash_webm_a_url = stream['url']
                     if 's' in stream:
-                        sig = self.__class__.decipher(self.js, stream['s'])
+                        sig = self.__class__.s_to_sig(self.js, stream['s'])
                         dash_webm_a_url += '&sig={}'.format(sig)
                     dash_webm_a_size = stream['clen']
             for stream in streams: # video
@@ -505,7 +505,7 @@ def prepare(self, **kwargs):
                         mimeType = 'video/mp4'
                         dash_url = stream['url']
                         if 's' in stream:
-                            sig = self.__class__.decipher(self.js, stream['s'])
+                            sig = self.__class__.s_to_sig(self.js, stream['s'])
                             dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
@@ -524,7 +524,7 @@ def prepare(self, **kwargs):
                         mimeType = 'video/webm'
                         dash_url = stream['url']
                         if 's' in stream:
-                            sig = self.__class__.decipher(self.js, stream['s'])
+                            sig = self.__class__.s_to_sig(self.js, stream['s'])
                             dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
@@ -573,7 +573,7 @@ def extract(self, **kwargs):
                 if not hasattr(self, 'js'):
                     self.js = get_content(self.html5player)
                 s = self.streams[stream_id]['s']
-                sig = self.__class__.decipher(self.js, s)
+                sig = self.__class__.s_to_sig(self.js, s)
                 src += '&sig={}'.format(sig)
 
             self.streams[stream_id]['src'] = [src]

From bcbe4e816a7f1de3afdacf49fa4163608c1a1452 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 28 Oct 2020 02:39:33 +0100
Subject: [PATCH 0992/1225] add a simple GitHub Actions workflow

---
 .github/workflows/python-package.yml | 39 ++++++++++++++++++++++++++++
 1 file changed, 39 insertions(+)
 create mode 100644 .github/workflows/python-package.yml

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
new file mode 100644
index 0000000000..96eefcc555
--- /dev/null
+++ b/.github/workflows/python-package.yml
@@ -0,0 +1,39 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ develop ]
+  pull_request:
+    branches: [ develop ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.5, 3.6, 3.7, 3.8, pypy3]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with unittest
+      run: |
+        make test

From 2a7fcdc8e9b28ba34148e131b737f7f2385b1d2d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 28 Oct 2020 03:12:06 +0100
Subject: [PATCH 0993/1225] migrate From Travis CI to GitHub Actions

---
 .github/workflows/python-package.yml |  2 +-
 .travis.yml                          | 22 ----------------------
 README.md                            |  2 +-
 3 files changed, 2 insertions(+), 24 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 96eefcc555..b3d50ff741 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package
+name: develop
 
 on:
   push:
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index eedbeeb283..0000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-# https://travis-ci.org/soimort/you-get
-language: python
-python:
-  - "3.4"
-  - "3.5"
-  - "3.6"
-  - "3.7"
-  - "3.8"
-  #- "nightly"  (flake8 not working in python 3.9 yet, module 'ast' has no attribute 'AugLoad')
-  - "pypy3"
-before_install:
-  - pip install flake8
-before_script:
-  - flake8 . --count --select=E9,F63,F72,F82 --show-source --statistics
-script: make test
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/43cd57826e88ed8f2152
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: never     # options: [always|never|change] default: always
diff --git a/README.md b/README.md
index 3429f9d83c..6adeeb35b9 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
 # You-Get
 
+[![Build Status](https://github.com/soimort/you-get/workflows/develop/badge.svg)](https://github.com/soimort/you-get/actions)
 [![PyPI version](https://img.shields.io/pypi/v/you-get.svg)](https://pypi.python.org/pypi/you-get/)
-[![Build Status](https://travis-ci.org/soimort/you-get.svg)](https://travis-ci.org/soimort/you-get)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 **NOTICE: Read [this](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) if you are looking for the conventional "Issues" tab.**

From 4f599121d2f89b6613084ff8fdaa2986955b69de Mon Sep 17 00:00:00 2001
From: WhoIsSure <whoissure@outlook.com>
Date: Mon, 9 Nov 2020 23:24:35 +0800
Subject: [PATCH 0994/1225] [pptv] fix: download pptv error, code 403 / 405
 #2832

---
 src/you_get/extractors/pptv.py | 158 +++------------------------------
 1 file changed, 14 insertions(+), 144 deletions(-)

diff --git a/src/you_get/extractors/pptv.py b/src/you_get/extractors/pptv.py
index dacd78e4c7..ef25ac4469 100644
--- a/src/you_get/extractors/pptv.py
+++ b/src/you_get/extractors/pptv.py
@@ -174,7 +174,7 @@ def make_url(stream):
     src = []
     for i, seg in enumerate(stream['segs']):
         url = 'http://{}/{}/{}?key={}&k={}'.format(host, i, rid, key, key_expr)
-        url += '&fpp.ver=1.3.0.4&type='
+        url += '&type=web.fpp'
         src.append(url)
     return src
 
@@ -189,17 +189,27 @@ class PPTV(VideoExtractor):
     ]
 
     def prepare(self, **kwargs):
+        headers = {
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
+                          "Chrome/69.0.3497.100 Safari/537.36"
+        }
+        self.vid = match1(self.url, r'https?://sports.pptv.com/vod/(\d+)/*')
         if self.url and not self.vid:
             if not re.match(r'https?://v.pptv.com/show/(\w+)\.html', self.url):
                 raise('Unknown url pattern')
-            page_content = get_content(self.url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"})
+            page_content = get_content(self.url, headers)
+
             self.vid = match1(page_content, r'webcfg\s*=\s*{"id":\s*(\d+)')
+            if not self.vid:
+                request = urllib.request.Request(self.url, headers=headers)
+                response = urllib.request.urlopen(request)
+                self.vid = match1(response.url, r'https?://sports.pptv.com/vod/(\d+)/*')
 
         if not self.vid:
             raise('Cannot find id')
         api_url = 'http://web-play.pptv.com/webplay3-0-{}.xml'.format(self.vid)
-        api_url += '?appplt=flp&appid=pptv.flashplayer.vod&appver=3.4.2.28&type=&version=4'
-        dom = parseString(get_content(api_url,{"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"}))
+        api_url += '?type=web.fpp&param=type=web.fpp&version=4'
+        dom = parseString(get_content(api_url, headers))
         self.title, m_items, m_streams, m_segs = parse_pptv_xml(dom)
         xml_streams = merge_meta(m_items, m_streams, m_segs)
         for stream_id in xml_streams:
@@ -212,146 +222,6 @@ def prepare(self, **kwargs):
                     'src': src
             }
 
-'''
-def constructKey(arg):
-
-    def str2hex(s):
-        r=""
-        for i in s[:8]:
-            t=hex(ord(i))[2:]
-            if len(t)==1:
-                t="0"+t
-            r+=t
-        for i in range(16):
-            r+=hex(int(15*random()))[2:]
-        return r
-
-    #ABANDONED  Because SERVER_KEY is static
-    def getkey(s):
-        #returns 1896220160
-        l2=[i for i in s]
-        l4=0
-        l3=0
-        while l4<len(l2):
-            l5=l2[l4]
-            l6=ord(l5)
-            l7=l6<<((l4%4)*8)
-            l3=l3^l7
-            l4+=1
-        return l3
-        pass
-
-    def rot(k,b): ##>>> in as3
-        if k>=0:
-            return k>>b
-        elif k<0:
-            return (2**32+k)>>b
-        pass
-
-    def lot(k,b):
-        return (k<<b)%(2**32)
-
-    #WTF?
-    def encrypt(arg1,arg2):
-        delta=2654435769
-        l3=16;
-        l4=getkey(arg2)  #1896220160
-        l8=[i for i in arg1]
-        l10=l4;
-        l9=[i for i in arg2]
-        l5=lot(l10,8)|rot(l10,24)#101056625
-        # assert l5==101056625
-        l6=lot(l10,16)|rot(l10,16)#100692230
-        # assert 100692230==l6
-        l7=lot(l10,24)|rot(l10,8)
-        # assert 7407110==l7
-        l11=""
-        l12=0
-        l13=ord(l8[l12])<<0
-        l14=ord(l8[l12+1])<<8
-        l15=ord(l8[l12+2])<<16
-        l16=ord(l8[l12+3])<<24
-        l17=ord(l8[l12+4])<<0
-        l18=ord(l8[l12+5])<<8
-        l19=ord(l8[l12+6])<<16
-        l20=ord(l8[l12+7])<<24
-
-        l21=(((0|l13)|l14)|l15)|l16
-        l22=(((0|l17)|l18)|l19)|l20
-
-        l23=0
-        l24=0
-        while l24<32:
-            l23=(l23+delta)%(2**32)
-            l33=(lot(l22,4)+l4)%(2**32)
-            l34=(l22+l23)%(2**32)
-            l35=(rot(l22,5)+l5)%(2**32)
-            l36=(l33^l34)^l35
-            l21=(l21+l36)%(2**32)
-            l37=(lot(l21,4)+l6)%(2**32)
-            l38=(l21+l23)%(2**32)
-            l39=(rot(l21,5))%(2**32)
-            l40=(l39+l7)%(2**32)
-            l41=((l37^l38)%(2**32)^l40)%(2**32)
-            l22=(l22+l41)%(2**32)
-
-            l24+=1
-
-        l11+=chr(rot(l21,0)&0xff)
-        l11+=chr(rot(l21,8)&0xff)
-        l11+=chr(rot(l21,16)&0xff)
-        l11+=chr(rot(l21,24)&0xff)
-        l11+=chr(rot(l22,0)&0xff)
-        l11+=chr(rot(l22,8)&0xff)
-        l11+=chr(rot(l22,16)&0xff)
-        l11+=chr(rot(l22,24)&0xff)
-
-        return l11
-
-
-    loc1=hex(int(arg))[2:]+(16-len(hex(int(arg))[2:]))*"\x00"
-    SERVER_KEY="qqqqqww"+"\x00"*9
-    res=encrypt(loc1,SERVER_KEY)
-    return str2hex(res)
-
-
-def pptv_download_by_id(id, title = None, output_dir = '.', merge = True, info_only = False):
-    xml = get_html('http://web-play.pptv.com/webplay3-0-%s.xml?type=web.fpp' % id)
-    #vt=3 means vod mode vt=5 means live mode
-    host = r1(r'<sh>([^<>]+)</sh>', xml)
-    k = r1(r'<key expire=[^<>]+>([^<>]+)</key>', xml)
-    rid = r1(r'rid="([^"]+)"', xml)
-    title = r1(r'nm="([^"]+)"', xml)
-
-    st=r1(r'<st>([^<>]+)</st>',xml)[:-4]
-    st=time.mktime(time.strptime(st))*1000-60*1000-time.time()*1000
-    st+=time.time()*1000
-    st=st/1000
-
-    key=constructKey(st)
-
-    pieces = re.findall('<sgm no="(\d+)"[^<>]+fs="(\d+)"', xml)
-    numbers, fs = zip(*pieces)
-    urls=["http://{}/{}/{}?key={}&fpp.ver=1.3.0.4&k={}&type=web.fpp".format(host,i,rid,key,k) for i in range(max(map(int,numbers))+1)]
-
-    total_size = sum(map(int, fs))
-    assert rid.endswith('.mp4')
-    print_info(site_info, title, 'mp4', total_size)
-
-    if not info_only:
-        try:
-            download_urls(urls, title, 'mp4', total_size, output_dir = output_dir, merge = merge)
-        except urllib.error.HTTPError:
-            #for key expired
-            pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
-
-def pptv_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
-    assert re.match(r'http://v.pptv.com/show/(\w+)\.html', url)
-    html = get_html(url)
-    id = r1(r'webcfg\s*=\s*{"id":\s*(\d+)', html)
-    assert id
-    pptv_download_by_id(id, output_dir = output_dir, merge = merge, info_only = info_only)
-'''
 site = PPTV()
 #site_info = "PPTV.com"
 #download = pptv_download

From 1b1f1dd1181bb15dabd04f928842891ac635f49c Mon Sep 17 00:00:00 2001
From: richard <yueyericardo@gmail.com>
Date: Mon, 9 Nov 2020 23:32:39 -0500
Subject: [PATCH 0995/1225] update regex to match vid for xinpianchang

---
 src/you_get/extractors/xinpianchang.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/xinpianchang.py b/src/you_get/extractors/xinpianchang.py
index fac3d01fd0..1121550c0f 100644
--- a/src/you_get/extractors/xinpianchang.py
+++ b/src/you_get/extractors/xinpianchang.py
@@ -20,7 +20,7 @@ class Xinpianchang(VideoExtractor):
     def prepare(self, **kwargs):
         # find key
         page_content = get_content(self.url)
-        match_rule = r"vid: \"(.+?)\","
+        match_rule = r"vid = \"(.+?)\";"
         key = re.findall(match_rule, page_content)[0]
 
         # get videos info

From b0aca8438c512586389f4bc29bf39c18dfd5b1ad Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 11 Nov 2020 23:54:42 +0100
Subject: [PATCH 0996/1225] [test] remove case for tiktok

---
 tests/test.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 00bd4cbb7e..26a0aa1894 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,7 +40,7 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_bilibil(self):
+    def test_bilibili(self):
         bilibili.download(
             "https://www.bilibili.com/watchlater/#/BV1PE411q7mZ/p6", info_only=True
         )
@@ -58,10 +58,10 @@ def test_soundcloud(self):
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
         #)
 
-    def tests_tiktok(self):
-        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
-        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
-        tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True)
+    #def tests_tiktok(self):
+    #    tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+    #    tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
+    #    tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True)
 
 
 if __name__ == '__main__':

From c074d8100110c1c2eed27f2f4ffb12f7670d6e74 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 12 Nov 2020 00:00:24 +0100
Subject: [PATCH 0997/1225] [common] update UA

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 515214079e..79fc74d1da 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -143,7 +143,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0',  # noqa
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/79.0.309.43',  # noqa
 }
 
 if sys.stdout.isatty():

From 67e9f7c6c1aa1d07eb2b8dbbf273cfa14d4dd21a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 12 Nov 2020 00:08:17 +0100
Subject: [PATCH 0998/1225] [test] remove a case for youtube that often fails

---
 tests/test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 26a0aa1894..0f7595b316 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -33,9 +33,9 @@ def test_youtube(self):
             'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
             info_only=True
         )
-        youtube.download(
-            'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
-        )
+        #youtube.download(
+        #    'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
+        #)
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)

From 4a9d2c1e13b8918deba39af515d315b60e545422 Mon Sep 17 00:00:00 2001
From: Kagamia <amethyst50504724@msn.com>
Date: Mon, 16 Nov 2020 00:25:29 +0800
Subject: [PATCH 0999/1225] add fake header

---
 src/you_get/extractors/netease.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index c7c0f66683..ca1be887b8 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -123,10 +123,10 @@ def netease_song_download(song, output_dir='.', info_only=False, playlist_prefix
                             output_dir=output_dir, info_only=info_only)
 
 def netease_download_common(title, url_best, output_dir, info_only):
-    songtype, ext, size = url_info(url_best)
+    songtype, ext, size = url_info(url_best, faker=True)
     print_info(site_info, title, songtype, size)
     if not info_only:
-        download_urls([url_best], title, ext, size, output_dir)
+        download_urls([url_best], title, ext, size, output_dir, faker=True)
 
 
 def netease_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):

From c9001c70e9c0e43b105b6fbf4996c128f1bda5c1 Mon Sep 17 00:00:00 2001
From: widtrizz <54760244+widtrizz@users.noreply.github.com>
Date: Tue, 17 Nov 2020 21:08:26 +0800
Subject: [PATCH 1000/1225] Update ccode 0590

---
 src/you_get/extractors/youku.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index d7107eca24..7f4be852d8 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -41,7 +41,6 @@ class Youku(VideoExtractor):
     mobile_ua = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
     dispatcher_url = 'vali.cp31.ott.cibntv.net'
 
-    # Last updated: 2017-10-13
     stream_types = [
         {'id': 'hd3',      'container': 'flv', 'video_profile': '1080P'},
         {'id': 'hd3v2',    'container': 'flv', 'video_profile': '1080P'},
@@ -78,7 +77,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0519'
+        self.ccode = '0590'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From 6f9cd8a069611e69d26da77eac676d33ca99c735 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 18 Nov 2020 16:35:59 +0100
Subject: [PATCH 1001/1225] version 0.4.1488

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index b469651960..0e6f123069 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1475'
+__version__ = '0.4.1488'

From 6e39a594e4b09b1379be649517dcf2d237ef6263 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Dec 2020 16:14:40 +0100
Subject: [PATCH 1002/1225] [instagram] fix extraction

---
 src/you_get/extractors/instagram.py | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 86905a77b2..6f6bf17351 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -9,7 +9,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     html = get_html(url)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
-    description = r1(r'<meta property="og:title" content="([^"]*)"', html)
+    description = r1(r'<title>\s([^<]*)</title>', html)
     title = "{} [{}]".format(description.replace("\n", " "), vid)
     stream = r1(r'<meta property="og:video" content="([^"]*)"', html)
     if stream:
@@ -19,11 +19,11 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if not info_only:
             download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
-        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
-        info = json.loads(data.group(1))
+        data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
+        post = json.loads(data.group(1))
 
-        if 'edge_sidecar_to_children' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
-            edges = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['edge_sidecar_to_children']['edges']
+        if 'edge_sidecar_to_children' in post['graphql']['shortcode_media']:
+            edges = post['graphql']['shortcode_media']['edge_sidecar_to_children']['edges']
             for edge in edges:
                 title = edge['node']['shortcode']
                 image_url = edge['node']['display_url']
@@ -40,10 +40,10 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                                   total_size=size,
                                   output_dir=output_dir)
         else:
-            title = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['shortcode']
-            image_url = info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['display_url']
-            if 'video_url' in info['entry_data']['PostPage'][0]['graphql']['shortcode_media']:
-                image_url =info['entry_data']['PostPage'][0]['graphql']['shortcode_media']['video_url']
+            title = post['graphql']['shortcode_media']['shortcode']
+            image_url = post['graphql']['shortcode_media']['display_url']
+            if 'video_url' in post['graphql']['shortcode_media']:
+                image_url = post['graphql']['shortcode_media']['video_url']
             ext = image_url.split('?')[0].split('.')[-1]
             size = int(get_head(image_url)['Content-Length'])
 

From 0247b06437a2161ff537169d5f49c2e7c0a95665 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 5 Dec 2020 17:19:27 +0100
Subject: [PATCH 1003/1225] [instagram] support extraction with cookies

---
 src/you_get/extractors/instagram.py | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 6f6bf17351..7c9d62725f 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -9,8 +9,10 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     html = get_html(url)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
-    description = r1(r'<title>\s([^<]*)</title>', html)
+    description = r1(r'<meta property="og:title" content="([^"]*)"', html) or \
+        r1(r'<title>\s([^<]*)</title>', html) # with logged-in cookies
     title = "{} [{}]".format(description.replace("\n", " "), vid)
+
     stream = r1(r'<meta property="og:video" content="([^"]*)"', html)
     if stream:
         _, ext, size = url_info(stream)
@@ -19,8 +21,16 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if not info_only:
             download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
-        data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
-        post = json.loads(data.group(1))
+        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
+        if data is not None:
+            info = json.loads(data.group(1))
+            post = info['entry_data']['PostPage'][0]
+        else:
+            # with logged-in cookies
+            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
+            if data is not None:
+                log.e('[Error] Cookies needed.')
+            post = json.loads(data.group(1))
 
         if 'edge_sidecar_to_children' in post['graphql']['shortcode_media']:
             edges = post['graphql']['shortcode_media']['edge_sidecar_to_children']['edges']

From dbb1296d548cf13312fa0706cbf212f42f064c94 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 9 Dec 2020 00:46:03 +0100
Subject: [PATCH 1004/1225] update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6adeeb35b9..ce412afd58 100644
--- a/README.md
+++ b/README.md
@@ -436,7 +436,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 火猫TV   | <http://www.huomao.com/>       |✓| | |
 | 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
-| 新片场 | <https://www.xinpianchang.com//>      |✓| | |
+| 新片场 | <https://www.xinpianchang.com/>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
 | TikTok | <https://www.tiktok.com/>      |✓| | |

From 4793e55e1465dbedacd04784974cfaf21bfaffb6 Mon Sep 17 00:00:00 2001
From: zhoudunguang <const_zhou@foxmail.com>
Date: Wed, 9 Dec 2020 16:04:03 +0800
Subject: [PATCH 1005/1225] update ccode 0532 for Youku

---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 7f4be852d8..92cdafb6ec 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -77,7 +77,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0590'
+        self.ccode = '0532'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From b00860e4beed7d72a20f212c5ab9f443b779dc2d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 11 Dec 2020 02:02:44 +0100
Subject: [PATCH 1006/1225] [instagram] add faker

---
 src/you_get/extractors/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 7c9d62725f..d5100a786d 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -6,7 +6,7 @@
 
 def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = r1(r'([^?]*)', url)
-    html = get_html(url)
+    html = get_html(url, faker=True)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
     description = r1(r'<meta property="og:title" content="([^"]*)"', html) or \

From 5c2bb3fa3448efeaf014009bd1a1daa43680683e Mon Sep 17 00:00:00 2001
From: Riceball LEE <snowyu.lee@gmail.com>
Date: Tue, 15 Dec 2020 21:41:41 +0800
Subject: [PATCH 1007/1225] feat: add lrts extractor

---
 README.md                      |  1 +
 src/you_get/common.py          |  1 +
 src/you_get/extractors/lrts.py | 70 ++++++++++++++++++++++++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 src/you_get/extractors/lrts.py

diff --git a/README.md b/README.md
index ce412afd58..6a23faf8c8 100644
--- a/README.md
+++ b/README.md
@@ -414,6 +414,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | 酷我音乐 | <http://www.kuwo.cn/>          | | |✓|
 | 乐视网   | <http://www.le.com/>           |✓| | |
 | 荔枝FM   | <http://www.lizhi.fm/>         | | |✓|
+| 懒人听书 | <http://www.lrts.me/>          | | |✓|
 | 秒拍     | <http://www.miaopai.com/>      |✓| | |
 | MioMio弹幕网 | <http://www.miomio.tv/>    |✓| | |
 | MissEvan<br/>猫耳FM | <http://www.missevan.com/> | | |✓|
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 79fc74d1da..2b6e05d23d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -76,6 +76,7 @@
     'letv'             : 'le',
     'lizhi'            : 'lizhi',
     'longzhu'          : 'longzhu',
+    'lrts'             : 'lrts',
     'magisto'          : 'magisto',
     'metacafe'         : 'metacafe',
     'mgtv'             : 'mgtv',
diff --git a/src/you_get/extractors/lrts.py b/src/you_get/extractors/lrts.py
new file mode 100644
index 0000000000..23abab5cfc
--- /dev/null
+++ b/src/you_get/extractors/lrts.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+__all__ = ['lrts_download']
+
+import logging
+from ..common import *
+
+def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    html = get_html(url)
+    matched = re.search(r"/book/(\d+)", url)
+    if not matched:
+        raise AssertionError("not found book number: %s" % url)
+    book_no = matched.group(1)
+    book_title = book_no
+    matched = re.search(r"<title>(.*)-(.*)</title>", html)
+    if matched:
+        book_title = matched.group(1)
+
+    matched = re.search(r"var totalCount='(\d+)'", html)
+    if not matched:
+        raise AssertionError("not found total count in html")
+    total_count = int(matched.group(1))
+    logging.debug('total: %s' % total_count)
+    page_size = 10
+    logging.debug('total page count: %s' % ((total_count // page_size) + 1))
+    headers = {
+      'Referer': url
+    }
+    items = []
+    if (total_count > page_size):
+        for page in range((total_count // page_size) + 1):
+            page_url = 'http://www.lrts.me/ajax/book/%s/%s/%s' % (book_no, page, page_size)
+            response_content = json.loads(post_content(page_url, headers))
+            if response_content['status'] != 'success':
+                raise AssertionError("got the page failed: %s" % (page_url))
+            data = response_content['data']['data']
+            if data:
+                for i in data:
+                    i['resName'] = parse.unquote(i['resName'])
+                items.extend(data)
+            else:
+                break
+
+    headers = {
+      'Referer': 'http://www.lrts.me/playlist'
+    }
+
+    for item in items:
+        i_url = 'http://www.lrts.me/ajax/path/4/%s/%s' % (item['fatherResId'], item['resId'])
+        response_content = json.loads(post_content(i_url, headers))
+        # logging.debug(response_content)
+        if response_content['status'] == 'success' and response_content['data']:
+            item['ok'] = True
+            item['url'] = response_content['data']
+
+    items = list(filter(lambda i: 'ok' in i and i['ok'], items))
+    print('Downloading %s: %s count ...' % (book_title, len(items)))
+
+    for item in items:
+        title = item['resName']
+        file_url = item['url']
+        # if not file_url: continue
+        _, _, size = url_info(file_url)
+        print_info(site_info, title, 'mp3', size)
+        if not info_only:
+            download_urls([file_url], title, 'mp3', size, output_dir, merge=merge)
+
+site_info = "lrts.me"
+download = lrts_download
+download_playlist = lrts_download

From e37836a40bd38feb7f2f616852883578b7153a6c Mon Sep 17 00:00:00 2001
From: Riceball LEE <snowyu.lee@gmail.com>
Date: Wed, 16 Dec 2020 10:31:52 +0800
Subject: [PATCH 1008/1225] feat: add arguments to specify the playlist first,
 last, page-size options

---
 src/you_get/common.py          | 17 ++++++++++++++++-
 src/you_get/extractors/lrts.py | 26 ++++++++++++++++++++------
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2b6e05d23d..7fe9d51dbf 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1557,6 +1557,21 @@ def print_version():
         '-l', '--playlist', action='store_true',
         help='Prefer to download a playlist'
     )
+
+    playlist_grp = parser.add_argument_group('Playlist optional options')
+    playlist_grp.add_argument(
+        '-first', '--first',  metavar='FIRST',
+        help='the first number'
+    )
+    playlist_grp.add_argument(
+        '-last', '--last',  metavar='LAST',
+        help='the last number'
+    )
+    playlist_grp.add_argument(
+        '-size', '--page-size',  metavar='PAGE_SIZE',
+        help='the page size number'
+    )
+
     download_grp.add_argument(
         '-a', '--auto-rename', action='store_true', default=False,
         help='Auto rename same name different files'
@@ -1674,7 +1689,7 @@ def print_version():
     socket.setdefaulttimeout(args.timeout)
 
     try:
-        extra = {}
+        extra = {'args': args}
         if extractor_proxy:
             extra['extractor_proxy'] = extractor_proxy
         if stream_id:
diff --git a/src/you_get/extractors/lrts.py b/src/you_get/extractors/lrts.py
index 23abab5cfc..d206491dff 100644
--- a/src/you_get/extractors/lrts.py
+++ b/src/you_get/extractors/lrts.py
@@ -4,15 +4,18 @@
 
 import logging
 from ..common import *
+from ..util import log, term
 
 def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
+    args = kwargs.get('args')
+    if not args: args = {}
     matched = re.search(r"/book/(\d+)", url)
     if not matched:
         raise AssertionError("not found book number: %s" % url)
     book_no = matched.group(1)
     book_title = book_no
-    matched = re.search(r"<title>(.*)-(.*)</title>", html)
+    matched = re.search(r"<title>([^-]*)[-](.*)[,](.*)</title>", html)
     if matched:
         book_title = matched.group(1)
 
@@ -20,15 +23,25 @@ def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if not matched:
         raise AssertionError("not found total count in html")
     total_count = int(matched.group(1))
-    logging.debug('total: %s' % total_count)
+    log.i('%s total: %s' % (book_title, total_count))
+    first_page = 0
+    if ('first' in args and args.first!= None):
+        first_page = int(args.first)
+
     page_size = 10
-    logging.debug('total page count: %s' % ((total_count // page_size) + 1))
+    if ('page_size' in args and args.page_size != None):
+        page_size = int(args.page_size)
+    last_page = (total_count // page_size) + 1
+    if ('last' in args and args.last != None):
+        last_page = int(args.last)
+
+    log.i('page size is %s, page from %s to %s' % (page_size, first_page, last_page))
     headers = {
       'Referer': url
     }
     items = []
     if (total_count > page_size):
-        for page in range((total_count // page_size) + 1):
+        for page in range(first_page, last_page):
             page_url = 'http://www.lrts.me/ajax/book/%s/%s/%s' % (book_no, page, page_size)
             response_content = json.loads(post_content(page_url, headers))
             if response_content['status'] != 'success':
@@ -48,13 +61,14 @@ def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     for item in items:
         i_url = 'http://www.lrts.me/ajax/path/4/%s/%s' % (item['fatherResId'], item['resId'])
         response_content = json.loads(post_content(i_url, headers))
-        # logging.debug(response_content)
+        logging.debug(response_content)
         if response_content['status'] == 'success' and response_content['data']:
             item['ok'] = True
             item['url'] = response_content['data']
+            logging.debug('ok')
 
     items = list(filter(lambda i: 'ok' in i and i['ok'], items))
-    print('Downloading %s: %s count ...' % (book_title, len(items)))
+    log.i('Downloading %s: %s count ...' % (book_title, len(items)))
 
     for item in items:
         title = item['resName']

From 17c740ae542e4ce2285da8775152cde8cbc782b8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 17 Dec 2020 02:01:23 +0100
Subject: [PATCH 1009/1225] [twitter] add faker

---
 src/you_get/extractors/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 602c18f65e..2346821147 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -34,7 +34,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    html = get_html(url, faker=False) # disable faker to prevent 302 infinite redirect
+    html = get_html(url, faker=True) # now it seems faker must be enabled
     screen_name = r1(r'twitter\.com/([^/]+)', url) or r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
     item_id = r1(r'twitter\.com/[^/]+/status/(\d+)', url) or r1(r'data-item-id="([^"]*)"', html) or \

From a5edbacada3974c3299c55b0e20465b9ab06a668 Mon Sep 17 00:00:00 2001
From: bUBBLE <www.1567891@qq.com>
Date: Fri, 18 Dec 2020 14:53:39 +0800
Subject: [PATCH 1010/1225] Update README.md

---
 README.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/README.md b/README.md
index ce412afd58..137cbab851 100644
--- a/README.md
+++ b/README.md
@@ -427,7 +427,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **Tudou<br/>土豆** | <http://www.tudou.com/> |✓| | |
 | 虾米     | <http://www.xiami.com/>        |✓| |✓|
 | 阳光卫视 | <http://www.isuntv.com/>       |✓| | |
-| **音悦Tai** | <http://www.yinyuetai.com/> |✓| | |
 | **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
 | 央视网   | <http://www.cntv.cn/>          |✓| | |

From 5a890eac531546f7bc737a0616a86262e418acdf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 21 Dec 2020 17:00:22 +0100
Subject: [PATCH 1011/1225] [soundcloud] fix

---
 src/you_get/extractors/soundcloud.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
index ecd3fc8d7e..08e9d5610e 100644
--- a/src/you_get/extractors/soundcloud.py
+++ b/src/you_get/extractors/soundcloud.py
@@ -19,7 +19,7 @@ def get_sndcd_apikey():
 def get_resource_info(resource_url, client_id):
     cont = get_content(resource_url, decoded=True)
 
-    x = re.escape('forEach(function(e){n(e)})}catch(t){}})},')
+    x = re.escape('forEach(function(e){n(e)})}catch(e){}})},')
     x = re.search(r'' + x + r'(.*)\);</script>', cont)
 
     info = json.loads(x.group(1))[-1]['data'][0]

From 6be1d0308ec52c212052b5808661636100ae2bfb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Dec 2020 18:45:55 +0100
Subject: [PATCH 1012/1225] [youtube] partial fix for #2857

---
 src/you_get/extractors/youtube.py | 47 ++++++++++++++++++++-----------
 1 file changed, 31 insertions(+), 16 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ac62e57c3a..9485b87668 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -203,28 +203,43 @@ def prepare(self, **kwargs):
                 # Parse video page (for DASH)
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
                 try:
-                    ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-
-                    # Workaround: get_video_info returns bad s. Why?
-                    if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
-                        stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
-                    else:
-                        stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
-                    #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
-
-                    if 'assets' in ytplayer_config:
-                        self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
-                    elif re.search('([^"]*/base\.js)"', video_page):
-                        self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
-                        self.html5player = self.html5player.replace('\/', '/') # unescape URL
-                    else:
-                        self.html5player = None
+                    try:
+                        # Complete ytplayer_config
+                        ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
+
+                        # Workaround: get_video_info returns bad s. Why?
+                        if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
+                            stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
+                        else:
+                            stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
+                        #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
+
+                        if 'assets' in ytplayer_config:
+                            self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
+                        elif re.search('([^"]*/base\.js)"', video_page):
+                            self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                            self.html5player = self.html5player.replace('\/', '/') # unescape URL
+                        else:
+                            self.html5player = None
+
+                    except:
+                        # ytplayer_config = {args:{raw_player_response:ytInitialPlayerResponse}}
+                        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+
+                        stream_list = ytInitialPlayerResponse['streamingData']['formats']
+                        #stream_list = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
+
+                        if re.search('([^"]*/base\.js)"', video_page):
+                            self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                        else:
+                            self.html5player = None
 
                 except:
                     if 'url_encoded_fmt_stream_map' not in video_info:
                         stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']
                     else:
                         stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
+
                     if re.search('([^"]*/base\.js)"', video_page):
                         self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
                     else:

From 9a4d9ef94e5a00666118d0c2c845c16266f7dbb5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Dec 2020 19:02:47 +0100
Subject: [PATCH 1013/1225] [youtube] fully fix #2857 with DASH stream

---
 src/you_get/extractors/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 9485b87668..30297c758d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -468,11 +468,16 @@ def prepare(self, **kwargs):
                                for afmt in video_info['adaptive_fmts'][0].split(',')]
                 else:
                     try:
-                        streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                        try:
+                            streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
+                        except:
+                            streams = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
                     except:  # no DASH stream at all
                         return
+
                     # streams without contentLength got broken urls, just remove them (#2767)
                     streams = [stream for stream in streams if 'contentLength' in stream]
+
                     for stream in streams:
                         stream['itag'] = str(stream['itag'])
                         if 'qualityLabel' in stream:

From 4f8c73e9ac27198e15694affb20a615fa9ffa48c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Dec 2020 19:19:28 +0100
Subject: [PATCH 1014/1225] [yinyuetai] purge (#2855)

---
 src/you_get/common.py               |  1 -
 src/you_get/extractors/__init__.py  |  3 +-
 src/you_get/extractors/embed.py     |  8 ------
 src/you_get/extractors/yinyuetai.py | 43 -----------------------------
 4 files changed, 1 insertion(+), 54 deletions(-)
 delete mode 100644 src/you_get/extractors/yinyuetai.py

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 79fc74d1da..224249b4ed 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -117,7 +117,6 @@
     'xiaojiadianvideo' : 'fc2video',
     'ximalaya'         : 'ximalaya',
     'xinpianchang'     : 'xinpianchang',
-    'yinyuetai'        : 'yinyuetai',
     'yizhibo'          : 'yizhibo',
     'youku'            : 'youku',
     'youtu'            : 'youtube',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 4280d236dd..8cf1ea7ce1 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -80,10 +80,9 @@
 from .wanmen import *
 from .xiami import *
 from .xinpianchang import *
-from .yinyuetai import *
 from .yixia import *
 from .youku import *
 from .youtube import *
 from .zhanqi import *
 from .zhibo import *
-from .zhihu import *
\ No newline at end of file
+from .zhihu import *
diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index ee53974729..aedf5137a2 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -13,7 +13,6 @@
 from .sina import sina_download_by_vid
 from .tudou import tudou_download_by_id
 from .vimeo import vimeo_download_by_id
-from .yinyuetai import yinyuetai_download_by_id
 from .youku import youku_download_by_vid
 from . import iqiyi
 from . import bokecc
@@ -40,8 +39,6 @@
 """
 tudou_api_patterns = [ ]
 
-yinyuetai_embed_patterns = [ 'player\.yinyuetai\.com/video/swf/(\d+)' ]
-
 iqiyi_embed_patterns = [ 'player\.video\.qiyi\.com/([^/]+)/[^/]+/[^/]+/[^/]+\.swf[^"]+tvId=(\d+)' ]
 
 netease_embed_patterns = [ '(http://\w+\.163\.com/movie/[^\'"]+)' ]
@@ -82,11 +79,6 @@ def embed_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         found = True
         tudou_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
 
-    vids = matchall(content, yinyuetai_embed_patterns)
-    for vid in vids:
-        found = True
-        yinyuetai_download_by_id(vid, title=title, output_dir=output_dir, merge=merge, info_only=info_only, **kwargs)
-
     vids = matchall(content, iqiyi_embed_patterns)
     for vid in vids:
         found = True
diff --git a/src/you_get/extractors/yinyuetai.py b/src/you_get/extractors/yinyuetai.py
deleted file mode 100644
index 6c39540fe1..0000000000
--- a/src/you_get/extractors/yinyuetai.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['yinyuetai_download', 'yinyuetai_download_by_id']
-
-from ..common import *
-
-def yinyuetai_download_by_id(vid, title=None, output_dir='.', merge=True, info_only=False):
-    video_info = json.loads(get_html('http://www.yinyuetai.com/insite/get-video-info?json=true&videoId=%s' % vid))
-    url_models = video_info['videoInfo']['coreVideoInfo']['videoUrlModels']
-    url_models = sorted(url_models, key=lambda i: i['qualityLevel'])
-    url = url_models[-1]['videoUrl']
-    type = ext = r1(r'\.(flv|mp4)', url)
-    _, _, size = url_info(url)
-
-    print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge = merge)
-
-def yinyuetai_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    id = r1(r'http://\w+.yinyuetai.com/video/(\d+)', url) or \
-         r1(r'http://\w+.yinyuetai.com/video/h5/(\d+)', url)
-    if not id:
-        yinyuetai_download_playlist(url, output_dir=output_dir, merge=merge, info_only=info_only)
-        return
-
-    html = get_html(url, 'utf-8')
-    title = r1(r'<meta property="og:title"\s+content="([^"]+)"/>', html) or r1(r'<title>(.*)', html)
-    assert title
-    title = parse.unquote(title)
-    title = escape_file_path(title)
-    yinyuetai_download_by_id(id, title, output_dir, merge=merge, info_only=info_only)
-
-def yinyuetai_download_playlist(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    playlist = r1(r'http://\w+.yinyuetai.com/playlist/(\d+)', url)
-    html = get_html(url)
-    data_ids = re.findall(r'data-index="\d+"\s*data-id=(\d+)', html)
-    for data_id in data_ids:
-        yinyuetai_download('http://v.yinyuetai.com/video/' + data_id,
-                           output_dir=output_dir, merge=merge, info_only=info_only)
-
-site_info = "YinYueTai.com"
-download = yinyuetai_download
-download_playlist = yinyuetai_download_playlist

From a2155e85f89b6315b3e3145e97c95b93b087951a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 26 Dec 2020 19:21:43 +0100
Subject: [PATCH 1015/1225] version 0.4.1500

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 0e6f123069..95505814c5 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1488'
+__version__ = '0.4.1500'

From 9432ce3c71d4f7df1a090a4c8defa76cd9ff06f3 Mon Sep 17 00:00:00 2001
From: Riceball LEE <snowyu.lee@gmail.com>
Date: Sun, 27 Dec 2020 15:45:21 +0800
Subject: [PATCH 1016/1225] fix(lrts): can not download audio for the count
 less than pagesize

---
 src/you_get/extractors/lrts.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/lrts.py b/src/you_get/extractors/lrts.py
index d206491dff..94d12a25d7 100644
--- a/src/you_get/extractors/lrts.py
+++ b/src/you_get/extractors/lrts.py
@@ -40,20 +40,18 @@ def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
       'Referer': url
     }
     items = []
-    if (total_count > page_size):
-        for page in range(first_page, last_page):
-            page_url = 'http://www.lrts.me/ajax/book/%s/%s/%s' % (book_no, page, page_size)
-            response_content = json.loads(post_content(page_url, headers))
-            if response_content['status'] != 'success':
-                raise AssertionError("got the page failed: %s" % (page_url))
-            data = response_content['data']['data']
-            if data:
-                for i in data:
-                    i['resName'] = parse.unquote(i['resName'])
-                items.extend(data)
-            else:
-                break
-
+    for page in range(first_page, last_page):
+        page_url = 'http://www.lrts.me/ajax/book/%s/%s/%s' % (book_no, page, page_size)
+        response_content = json.loads(post_content(page_url, headers))
+        if response_content['status'] != 'success':
+            raise AssertionError("got the page failed: %s" % (page_url))
+        data = response_content['data']['data']
+        if data:
+            for i in data:
+                i['resName'] = parse.unquote(i['resName'])
+            items.extend(data)
+        else:
+            break
     headers = {
       'Referer': 'http://www.lrts.me/playlist'
     }
@@ -61,7 +59,6 @@ def lrts_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     for item in items:
         i_url = 'http://www.lrts.me/ajax/path/4/%s/%s' % (item['fatherResId'], item['resId'])
         response_content = json.loads(post_content(i_url, headers))
-        logging.debug(response_content)
         if response_content['status'] == 'success' and response_content['data']:
             item['ok'] = True
             item['url'] = response_content['data']

From 027130a45ab4c69ab3b1ba1bff9dd9b34ada5a4f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Dec 2020 17:22:25 +0100
Subject: [PATCH 1017/1225] [common] use only double-dashed arguments

---
 src/you_get/common.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 268448d886..83dfa2ac66 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1559,15 +1559,15 @@ def print_version():
 
     playlist_grp = parser.add_argument_group('Playlist optional options')
     playlist_grp.add_argument(
-        '-first', '--first',  metavar='FIRST',
+        '--first', metavar='FIRST',
         help='the first number'
     )
     playlist_grp.add_argument(
-        '-last', '--last',  metavar='LAST',
+        '--last', metavar='LAST',
         help='the last number'
     )
     playlist_grp.add_argument(
-        '-size', '--page-size',  metavar='PAGE_SIZE',
+        '--size', '--page-size', metavar='PAGE_SIZE',
         help='the page size number'
     )
 

From 15cc69a120cd297f2a12c5000eb1d9717e3105f4 Mon Sep 17 00:00:00 2001
From: Bochun Bai <bbc@sinofool.net>
Date: Wed, 6 Jan 2021 20:59:13 -0800
Subject: [PATCH 1018/1225] Fix mgtv: adds tk2 and referer

---
 src/you_get/extractors/mgtv.py | 114 +++++++++++++++++++++++----------
 1 file changed, 79 insertions(+), 35 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 657167a65f..c8a0306512 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -9,19 +9,36 @@
 from os.path import dirname
 import re
 
+import base64
+import time
+import uuid
+
+
 class MGTV(VideoExtractor):
     name = "芒果 (MGTV)"
 
     # Last updated: 2016-11-13
     stream_types = [
+        {'id': 'fhd', 'container': 'ts', 'video_profile': '蓝光'},
         {'id': 'hd', 'container': 'ts', 'video_profile': '超清'},
         {'id': 'sd', 'container': 'ts', 'video_profile': '高清'},
         {'id': 'ld', 'container': 'ts', 'video_profile': '标清'},
     ]
-    
-    id_dic = {i['video_profile']:(i['id']) for i in stream_types}
-    
-    api_endpoint = 'http://pcweb.api.mgtv.com/player/video?video_id={video_id}'
+
+    id_dic = {i['video_profile']: (i['id']) for i in stream_types}
+
+    did = str(uuid.uuid4())
+    ver = '0.3.0301'
+    pno = '1030'
+
+    def tk2(self):
+        return base64.urlsafe_b64encode(b'did=%s|ver=%s|pno=%s|clit=%d' % (
+            self.did.encode(), self.ver.encode(), self.pno.encode(), time.time())).decode('utf-8')[::-1]
+
+    info_endpoint = 'https://pcweb.api.mgtv.com/video/info?vid={video_id}'
+    player_endpoint = 'https://pcweb.api.mgtv.com/player/video?did={did}&tk2={tk2}&video_id={video_id}'
+    source_endpoint = 'https://pcweb.api.mgtv.com/player/getSource?tk2={tk2}&pm2={pm2}&video_id={video_id}'
+    playlist_endpoint = 'https://pcweb.api.mgtv.com/episode/list?video_id={video_id}&page={page}&size=30'
 
     @staticmethod
     def get_vid_from_url(url):
@@ -31,71 +48,95 @@ def get_vid_from_url(url):
         if not vid:
             vid = match1(url, 'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
         return vid
-    
-    #----------------------------------------------------------------------
-    @staticmethod
-    def get_mgtv_real_url(url):
+
+    # ----------------------------------------------------------------------
+    def get_mgtv_real_url(self, url):
         """str->list of str
         Give you the real URLs."""
         content = loads(get_content(url))
         m3u_url = content['info']
         split = urlsplit(m3u_url)
-        
-        base_url = "{scheme}://{netloc}{path}/".format(scheme = split[0],
-                                                      netloc = split[1],
-                                                      path = dirname(split[2]))
 
-        content = get_content(content['info'])  #get the REAL M3U url, maybe to be changed later?
+        base_url = "{scheme}://{netloc}{path}/".format(scheme=split[0],
+                                                       netloc=split[1],
+                                                       path=dirname(split[2]))
+
+        content = get_content(content['info'],
+                              headers={'Referer': self.url})  # get the REAL M3U url, maybe to be changed later?
         segment_list = []
         segments_size = 0
         for i in content.split():
-            if not i.startswith('#'):  #not the best way, better we use the m3u8 package
+            if not i.startswith('#'):  # not the best way, better we use the m3u8 package
                 segment_list.append(base_url + i)
             # use ext-info for fast size calculate
             elif i.startswith('#EXT-MGTV-File-SIZE:'):
-                segments_size += int(i[i.rfind(':')+1:])
+                segments_size += int(i[i.rfind(':') + 1:])
 
         return m3u_url, segments_size, segment_list
 
     def download_playlist_by_url(self, url, **kwargs):
-        pass
+        self.url = url
+        self.vid = self.get_vid_from_url(self.url)
+        content_playlist = get_content(self.playlist_endpoint.format(video_id=self.vid, page=1))
+        content_playlist = loads(content_playlist)
+        for ep in content_playlist['data']['list']:
+            self.download_by_url('https://www.mgtv.com' + ep['url'], **kwargs)
+        max_page = content_playlist['data']['total_page']
+        for page in range(2, max_page + 1):
+            content_playlist = get_content(self.playlist_endpoint.format(video_id=self.vid, page=page))
+            content_playlist = loads(content_playlist)
+            for ep in content_playlist['data']['list']:
+                self.download_by_url('https://www.mgtv.com' + ep['url'], **kwargs)
 
     def prepare(self, **kwargs):
         if self.url:
             self.vid = self.get_vid_from_url(self.url)
-        content = get_content(self.api_endpoint.format(video_id = self.vid))
-        content = loads(content)
-        self.title = content['data']['info']['title']
-        domain = content['data']['stream_domain'][0]
-        
-        #stream_available = [i['name'] for i in content['data']['stream']]
+        content_info = get_content(self.info_endpoint.format(video_id=self.vid))
+        log.d(content_info)
+        content_info = loads(content_info)
+        self.title = content_info['data']['info']['videoName']
+
+        content_player = get_content(self.player_endpoint.format(did=self.did, video_id=self.vid, tk2=self.tk2()))
+        log.d(content_player)
+        content_player = loads(content_player)
+        pm2 = content_player['data']['atc']['pm2']
+
+        content_source = get_content(self.source_endpoint.format(video_id=self.vid, tk2=self.tk2(), pm2=pm2))
+        log.d(content_source)
+        content_source = loads(content_source)
+        domain = content_source['data']['stream_domain'][0]
+
+        # stream_available = [i['name'] for i in content['data']['stream']]
         stream_available = {}
-        for i in content['data']['stream']:
+        for i in content_source['data']['stream']:
             stream_available[i['name']] = i['url']
 
         for s in self.stream_types:
             if s['video_profile'] in stream_available.keys():
                 quality_id = self.id_dic[s['video_profile']]
                 url = stream_available[s['video_profile']]
-                url = domain + re.sub( r'(\&arange\=\d+)', '', url)  #Un-Hum
+                if url is None or url == '':
+                    # skip invalid profile with empty url
+                    continue
+                url = domain + re.sub(r'(\&arange\=\d+)', '', url)  # Un-Hum
                 m3u8_url, m3u8_size, segment_list_this = self.get_mgtv_real_url(url)
 
                 stream_fileid_list = []
                 for i in segment_list_this:
                     stream_fileid_list.append(os.path.basename(i).split('.')[0])
 
-            #make pieces
+            # make pieces
             pieces = []
             for i in zip(stream_fileid_list, segment_list_this):
-                pieces.append({'fileid': i[0], 'segs': i[1],})
+                pieces.append({'fileid': i[0], 'segs': i[1], })
 
                 self.streams[quality_id] = {
-                        'container': s['container'],
-                        'video_profile': s['video_profile'],
-                        'size': m3u8_size,
-                        'pieces': pieces,
-                        'm3u8_url': m3u8_url
-                    }
+                    'container': s['container'],
+                    'video_profile': s['video_profile'],
+                    'size': m3u8_size,
+                    'pieces': pieces,
+                    'm3u8_url': m3u8_url
+                }
 
             if not kwargs['info_only']:
                 self.streams[quality_id]['src'] = segment_list_this
@@ -132,7 +173,8 @@ def download(self, **kwargs):
                 if 'index' not in kwargs:
                     self.p([])
                 else:
-                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else self.streams_sorted[0]['itag']
+                    stream_id = self.streams_sorted[0]['id'] if 'id' in self.streams_sorted[0] else \
+                        self.streams_sorted[0]['itag']
                     self.p_i(stream_id)
 
         # default to use the best quality
@@ -148,8 +190,10 @@ def download(self, **kwargs):
             else:
                 download_urls(stream_info['src'], self.title, stream_info['container'], stream_info['size'],
                               output_dir=kwargs['output_dir'],
-                              merge=kwargs.get('merge', True))
-                              # av=stream_id in self.dash_streams)
+                              merge=kwargs.get('merge', True),
+                              headers={'Referer': self.url})
+                # av=stream_id in self.dash_streams)
+
 
 site = MGTV()
 download = site.download_by_url

From e914a4bbbfefe9db807dd55416a0e5bb7b467536 Mon Sep 17 00:00:00 2001
From: Bochun Bai <bbc@sinofool.net>
Date: Wed, 13 Jan 2021 20:55:19 -0800
Subject: [PATCH 1019/1225] Mgtv: add another format of url Fix indentation
 causing undefined variable

---
 src/you_get/extractors/mgtv.py | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index c8a0306512..053212efa0 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -47,6 +47,8 @@ def get_vid_from_url(url):
         vid = match1(url, 'https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
         if not vid:
             vid = match1(url, 'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
+        if not vid:
+            vid = match1(url, 'https?://www.mgtv.com/s/(\d+).html')
         return vid
 
     # ----------------------------------------------------------------------
@@ -125,21 +127,21 @@ def prepare(self, **kwargs):
                 for i in segment_list_this:
                     stream_fileid_list.append(os.path.basename(i).split('.')[0])
 
-            # make pieces
-            pieces = []
-            for i in zip(stream_fileid_list, segment_list_this):
-                pieces.append({'fileid': i[0], 'segs': i[1], })
-
-                self.streams[quality_id] = {
-                    'container': s['container'],
-                    'video_profile': s['video_profile'],
-                    'size': m3u8_size,
-                    'pieces': pieces,
-                    'm3u8_url': m3u8_url
-                }
-
-            if not kwargs['info_only']:
-                self.streams[quality_id]['src'] = segment_list_this
+                # make pieces
+                pieces = []
+                for i in zip(stream_fileid_list, segment_list_this):
+                    pieces.append({'fileid': i[0], 'segs': i[1], })
+
+                    self.streams[quality_id] = {
+                        'container': s['container'],
+                        'video_profile': s['video_profile'],
+                        'size': m3u8_size,
+                        'pieces': pieces,
+                        'm3u8_url': m3u8_url
+                    }
+
+                if not kwargs['info_only']:
+                    self.streams[quality_id]['src'] = segment_list_this
 
     def extract(self, **kwargs):
         if 'stream_id' in kwargs and kwargs['stream_id']:

From 81d416d1cdb1e43ceaef2fb5d7c6905c14b4aa32 Mon Sep 17 00:00:00 2001
From: Zhenyao She <dr.tobyshe@gmail.com>
Date: Sat, 16 Jan 2021 11:02:41 -0500
Subject: [PATCH 1020/1225] ignore vim swap files

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 99b18775ea..716c13b304 100644
--- a/.gitignore
+++ b/.gitignore
@@ -84,6 +84,7 @@ _*
 *.m4a
 *.DS_Store
 *.txt
+*.sw[a-p]
 
 *.zip
 

From 240b7a81118ac99055d9311a3a0bf3ff8fddc256 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 18 Jan 2021 18:37:24 +0100
Subject: [PATCH 1021/1225] [youtube] fix extraction of caption tracks, close
 #2866

---
 src/you_get/extractors/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 30297c758d..e954b1f4c2 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -353,7 +353,10 @@ def prepare(self, **kwargs):
 
         # Prepare caption tracks
         try:
-            caption_tracks = json.loads(ytplayer_config['args']['player_response'])['captions']['playerCaptionsTracklistRenderer']['captionTracks']
+            try:
+                caption_tracks = json.loads(ytplayer_config['args']['player_response'])['captions']['playerCaptionsTracklistRenderer']['captionTracks']
+            except:
+                caption_tracks = ytInitialPlayerResponse['captions']['playerCaptionsTracklistRenderer']['captionTracks']
             for ct in caption_tracks:
                 ttsurl, lang = ct['baseUrl'], ct['languageCode']
 

From 8b473e725662bfff520d27c2eb1589d46fcafda5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 6 Feb 2021 18:04:09 +0100
Subject: [PATCH 1022/1225] [xiami] xiami is dead

---
 README.md                          |   1 -
 src/you_get/common.py              |   1 -
 src/you_get/extractors/__init__.py |   1 -
 src/you_get/extractors/xiami.py    | 215 -----------------------------
 4 files changed, 218 deletions(-)
 delete mode 100644 src/you_get/extractors/xiami.py

diff --git a/README.md b/README.md
index bd2f02bd79..d2fdaa99c0 100644
--- a/README.md
+++ b/README.md
@@ -426,7 +426,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
 | Sohu<br/>搜狐视频 | <http://tv.sohu.com/> |✓| | |
 | **Tudou<br/>土豆** | <http://www.tudou.com/> |✓| | |
-| 虾米     | <http://www.xiami.com/>        |✓| |✓|
 | 阳光卫视 | <http://www.isuntv.com/>       |✓| | |
 | **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |
 | 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index 83dfa2ac66..67ef581b1b 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -113,7 +113,6 @@
     'veoh'             : 'veoh',
     'vine'             : 'vine',
     'vk'               : 'vk',
-    'xiami'            : 'xiami',
     'xiaokaxiu'        : 'yixia',
     'xiaojiadianvideo' : 'fc2video',
     'ximalaya'         : 'ximalaya',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 8cf1ea7ce1..8c43a8bca3 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -78,7 +78,6 @@
 from .vk import *
 from .w56 import *
 from .wanmen import *
-from .xiami import *
 from .xinpianchang import *
 from .yixia import *
 from .youku import *
diff --git a/src/you_get/extractors/xiami.py b/src/you_get/extractors/xiami.py
deleted file mode 100644
index 16656adb17..0000000000
--- a/src/you_get/extractors/xiami.py
+++ /dev/null
@@ -1,215 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-__all__ = ['xiami_download']
-
-from ..common import *
-
-from xml.dom.minidom import parseString
-from urllib import parse
-
-def location_dec(str):
-    head = int(str[0])
-    str = str[1:]
-    rows = head
-    cols = int(len(str)/rows) + 1
-
-    out = ""
-    full_row = len(str) % head
-    for c in range(cols):
-        for r in range(rows):
-            if c == (cols - 1) and r >= full_row:
-                continue
-            if r < full_row:
-                char = str[r*cols+c]
-            else:
-                char = str[cols*full_row+(r-full_row)*(cols-1)+c]
-            out += char
-    return parse.unquote(out).replace("^", "0")
-
-def xiami_download_lyric(lrc_url, file_name, output_dir):
-    lrc = get_content(lrc_url, headers=fake_headers)
-    filename = get_filename(file_name)
-    if len(lrc) > 0:
-        with open(output_dir + "/" + filename + '.lrc', 'w', encoding='utf-8') as x:
-            x.write(lrc)
-
-def xiami_download_pic(pic_url, file_name, output_dir):
-    from ..util.strings import get_filename
-    pic_url = pic_url.replace('_1', '')
-    pos = pic_url.rfind('.')
-    ext = pic_url[pos:]
-    pic = get_content(pic_url, headers=fake_headers, decoded=False)
-    if len(pic) > 0:
-        with open(output_dir + "/" + file_name.replace('/', '-') + ext, 'wb') as x:
-            x.write(pic)
-
-def xiami_download_song(sid, output_dir = '.', info_only = False):
-    xml = get_content('http://www.xiami.com/song/playlist/id/%s/object_name/default/object_id/0' % sid, headers=fake_headers)
-    doc = parseString(xml)
-    i = doc.getElementsByTagName("track")[0]
-    artist = i.getElementsByTagName("artist")[0].firstChild.nodeValue
-    album_name = i.getElementsByTagName("album_name")[0].firstChild.nodeValue
-    song_title = i.getElementsByTagName("name")[0].firstChild.nodeValue
-    url = location_dec(i.getElementsByTagName("location")[0].firstChild.nodeValue)
-    try:
-        lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
-    except:
-        pass
-    type_, ext, size = url_info(url, headers=fake_headers)
-    if not ext:
-        ext = 'mp3'
-
-    print_info(site_info, song_title, ext, size)
-    if not info_only:
-        file_name = "%s - %s - %s" % (song_title, artist, album_name)
-        download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
-        try:
-            xiami_download_lyric(lrc_url, file_name, output_dir)
-        except:
-            pass
-
-def xiami_download_showcollect(cid, output_dir = '.', info_only = False):
-    html = get_content('http://www.xiami.com/song/showcollect/id/' + cid, headers=fake_headers)
-    collect_name = r1(r'<title>(.*)</title>', html)
-
-    xml = get_content('http://www.xiami.com/song/playlist/id/%s/type/3' % cid, headers=fake_headers)
-    doc = parseString(xml)
-    output_dir =  output_dir + "/" + "[" + collect_name + "]"
-    tracks = doc.getElementsByTagName("track")
-    track_nr = 1
-    for i in tracks:
-        artist=album_name=song_title=url=""
-        try:
-            song_id = i.getElementsByTagName("song_id")[0].firstChild.nodeValue
-            artist = i.getElementsByTagName("artist")[0].firstChild.nodeValue
-            album_name = i.getElementsByTagName("album_name")[0].firstChild.nodeValue
-            song_title = i.getElementsByTagName("title")[0].firstChild.nodeValue
-            url = location_dec(i.getElementsByTagName("location")[0].firstChild.nodeValue)
-        except:
-            log.e("Song %s failed. [Info Missing] artist:%s, album:%s, title:%s, url:%s" % (song_id, artist, album_name, song_title, url))
-            continue
-        try:
-            lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
-        except:
-            pass
-        type_, ext, size = url_info(url, headers=fake_headers)
-        if not ext:
-            ext = 'mp3'
-
-        print_info(site_info, song_title, ext, size)
-        if not info_only:
-            file_name = "%02d.%s - %s - %s" % (track_nr, song_title, artist, album_name)
-            download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
-            try:
-                xiami_download_lyric(lrc_url, file_name, output_dir)
-            except:
-                pass
-
-        track_nr += 1
-
-def xiami_download_album(aid, output_dir='.', info_only=False):
-    xml = get_content('http://www.xiami.com/song/playlist/id/%s/type/1' % aid, headers=fake_headers)
-    album_name = r1(r'<album_name><!\[CDATA\[(.*)\]\]>', xml)
-    artist = r1(r'<artist><!\[CDATA\[(.*)\]\]>', xml)
-    doc = parseString(xml)
-    output_dir = output_dir + "/%s - %s" % (artist, album_name)
-    track_list = doc.getElementsByTagName('trackList')[0]
-    tracks = track_list.getElementsByTagName("track")
-    track_nr = 1
-    pic_exist = False
-    for i in tracks:
-#in this xml track tag is used for both "track in a trackList" and track no
-#dirty here
-        if i.firstChild.nodeValue is not None:
-            continue
-        song_title = i.getElementsByTagName("songName")[0].firstChild.nodeValue
-        url = location_dec(i.getElementsByTagName("location")[0].firstChild.nodeValue)
-        try:
-            lrc_url = i.getElementsByTagName("lyric")[0].firstChild.nodeValue
-        except:
-            pass
-        if not pic_exist:
-            pic_url = i.getElementsByTagName("pic")[0].firstChild.nodeValue
-        type_, ext, size = url_info(url, headers=fake_headers)
-        if not ext:
-            ext = 'mp3'
-
-        print_info(site_info, song_title, ext, size)
-        if not info_only:
-            file_name = "%02d.%s" % (track_nr, song_title)
-            download_urls([url], file_name, ext, size, output_dir, headers=fake_headers)
-            try:
-                xiami_download_lyric(lrc_url, file_name, output_dir)
-            except:
-                pass
-            if not pic_exist:
-                xiami_download_pic(pic_url, 'cover', output_dir)
-                pic_exist = True
-
-        track_nr += 1
-
-def xiami_download_mv(url, output_dir='.', merge=True, info_only=False):
-    # FIXME: broken merge
-    page = get_content(url, headers=fake_headers)
-    title = re.findall('<title>([^<]+)', page)[0]
-    vid, uid = re.findall(r'vid:"(\d+)",uid:"(\d+)"', page)[0]
-    api_url = 'http://cloud.video.taobao.com/videoapi/info.php?vid=%s&uid=%s' % (vid, uid)
-    result = get_content(api_url, headers=fake_headers)
-    doc = parseString(result)
-    video_url = doc.getElementsByTagName("video_url")[-1].firstChild.nodeValue
-    length = int(doc.getElementsByTagName("length")[-1].firstChild.nodeValue)
-
-    v_urls = []
-    k_start = 0
-    total_size = 0
-    while True:
-        k_end = k_start + 20000000
-        if k_end >= length: k_end = length - 1
-        v_url = video_url + '/start_%s/end_%s/1.flv' % (k_start, k_end)
-        try:
-            _, ext, size = url_info(v_url)
-        except:
-            break
-        v_urls.append(v_url)
-        total_size += size
-        k_start = k_end + 1
-
-    print_info(site_info, title, ext, total_size)
-    if not info_only:
-        download_urls(v_urls, title, ext, total_size, output_dir, merge=merge, headers=fake_headers)
-
-def xiami_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-#albums
-    if re.match(r'http://www.xiami.com/album/\d+', url):
-        id = r1(r'http://www.xiami.com/album/(\d+)', url)
-        xiami_download_album(id, output_dir, info_only)
-    elif re.match(r'http://www.xiami.com/album/\w+', url):
-        page = get_content(url, headers=fake_headers)
-        album_id = re.search(r'rel="canonical"\s+href="http://www.xiami.com/album/([^"]+)"', page).group(1)
-        xiami_download_album(album_id, output_dir, info_only)
-
-#collections
-    if re.match(r'http://www.xiami.com/collect/\d+', url):
-        id = r1(r'http://www.xiami.com/collect/(\d+)', url)
-        xiami_download_showcollect(id, output_dir, info_only)
-
-#single track
-    if re.match(r'http://www.xiami.com/song/\d+\b', url):
-        id = r1(r'http://www.xiami.com/song/(\d+)', url)
-        xiami_download_song(id, output_dir, info_only)
-    elif re.match(r'http://www.xiami.com/song/\w+', url):
-        html = get_content(url, headers=fake_headers)
-        id = r1(r'rel="canonical" href="http://www.xiami.com/song/([^"]+)"', html)
-        xiami_download_song(id, output_dir, info_only)
-
-    if re.match('http://www.xiami.com/song/detail/id/\d+', url):
-        id = r1(r'http://www.xiami.com/song/detail/id/(\d+)', url)
-        xiami_download_song(id, output_dir, info_only)
-
-    if re.match('http://www.xiami.com/mv', url):
-        xiami_download_mv(url, output_dir, merge=merge, info_only=info_only)
-
-site_info = "Xiami.com"
-download = xiami_download
-download_playlist = playlist_not_supported("xiami")

From 92ac149d544a39821528c6882dc63ae08be5147e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 7 Feb 2021 14:11:03 +0100
Subject: [PATCH 1023/1225] [tumblr] fix extraction

---
 src/you_get/extractors/tumblr.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 9a314c7f03..1fdfcad06f 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -14,7 +14,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         return
 
     import ssl
-    ssl_context = request.HTTPSHandler(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1))
+    ssl_context = request.HTTPSHandler(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) # server requires TLS v1.2
     cookie_handler = request.HTTPCookieProcessor()
     opener = request.build_opener(ssl_context, cookie_handler)
     request.install_opener(opener)
@@ -45,23 +45,30 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
                      r1(r'<title>([^<\n]*)', html)
         urls = re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.jpg)', html) +\
                re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.png)', html) +\
-               re.findall(r'(https?://[^;"&]+/tumblr_[^";&]+_\d+\.gif)', html)
+               re.findall(r'(https?://[^;"&]+/tumblr_[^;"&]+_\d+\.gif)', html) +\
+               re.findall(r'(https?://\d+\.media\.tumblr\.com/[^;"&]+/s\d+x\d+/[^;"&]+\.jpg)', html) +\
+               re.findall(r'(https?://\d+\.media\.tumblr\.com/[^;"&]+/s\d+x\d+/[^;"&]+\.png)', html) +\
+               re.findall(r'(https?://\d+\.media\.tumblr\.com/[^;"&]+/s\d+x\d+/[^;"&]+\.gif)', html)
 
         tuggles = {}
         for url in urls:
             if url.endswith('.gif'):
                 hd_url = url
             elif url.endswith('.jpg'):
-                hd_url = r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg' # FIXME: decide actual quality
+                hd_url = url  # FIXME: decide actual quality # r1(r'(.+)_\d+\.jpg$', url) + '_1280.jpg'
             elif url.endswith('.png'):
-                hd_url = r1(r'(.+)_\d+\.png$', url) + '_1280.png' # FIXME: decide actual quality
+                hd_url = url  # FIXME: decide actual quality # r1(r'(.+)_\d+\.png$', url) + '_1280.png'
             else:
                 continue
             filename = parse.unquote(hd_url.split('/')[-1])
             title = '.'.join(filename.split('.')[:-1])
-            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title)
-            quality = int(r1(r'^tumblr_.+_(\d+)$', title))
+            tumblr_id = r1(r'^tumblr_(.+)_\d+$', title) or title
+            try:
+                quality = int(r1(r'^tumblr_.+_(\d+)$', title))
+            except:
+                quality = int(r1(r'/s(\d+)x\d+/', hd_url))
             ext = filename.split('.')[-1]
+
             try:
                 size = int(get_head(hd_url)['Content-Length'])
                 if tumblr_id not in tuggles or tuggles[tumblr_id]['quality'] < quality:

From 980ba1bc2e3c41b8c41963cdfd05bbc1b3de9801 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 8 Feb 2021 17:35:23 +0100
Subject: [PATCH 1024/1225] [instagram] data not none even when missing cookies

---
 src/you_get/extractors/instagram.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index d5100a786d..0a9da3458f 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -22,14 +22,15 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
         data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
-        if data is not None:
+        try:
             info = json.loads(data.group(1))
             post = info['entry_data']['PostPage'][0]
-        else:
+            assert post
+        except:
             # with logged-in cookies
             data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
             if data is not None:
-                log.e('[Error] Cookies needed.')
+                log.e('[Warning] Cookies needed.')
             post = json.loads(data.group(1))
 
         if 'edge_sidecar_to_children' in post['graphql']['shortcode_media']:

From 3ab931a6a0fc2fc95d18865725fc236c4f53b5e1 Mon Sep 17 00:00:00 2001
From: v4hn <me@v4hn.de>
Date: Wed, 17 Feb 2021 14:10:15 +0100
Subject: [PATCH 1025/1225] [youtube] fix playlist extraction & comprehensive
 error output

---
 src/you_get/extractors/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index e954b1f4c2..7e4c74f071 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -157,7 +157,12 @@ def download_playlist_by_url(self, url, **kwargs):
             log.wtf('[Failed] Unsupported URL pattern.')
 
         video_page = get_content('https://www.youtube.com/playlist?list=%s' % playlist_id)
-        ytInitialData = json.loads(match1(video_page, r'window\["ytInitialData"\]\s*=\s*(.+);'))
+        playlist_json_serialized = match1(video_page, r'window\["ytInitialData"\]\s*=\s*(.+);', r'var\s+ytInitialData\s*=\s*([^;]+);')
+
+        if len(playlist_json_serialized) == 0:
+            log.wtf('[Failed] Unable to extract playlist data')
+
+        ytInitialData = json.loads(playlist_json_serialized[0])
 
         tab0 = ytInitialData['contents']['twoColumnBrowseResultsRenderer']['tabs'][0]
         itemSection0 = tab0['tabRenderer']['content']['sectionListRenderer']['contents'][0]

From 28af439fcadcd37e6171baddda433ffa0fe5a684 Mon Sep 17 00:00:00 2001
From: SHMoney2021 <15300607716@163.com>
Date: Mon, 8 Mar 2021 08:52:43 +0800
Subject: [PATCH 1026/1225] fix page count error in 'space_video' case

---
 src/you_get/extractors/bilibili.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index cdcccf2063..a812d72df3 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -4,6 +4,8 @@
 from ..extractor import VideoExtractor
 
 import hashlib
+import math
+
 
 class Bilibili(VideoExtractor):
     name = "Bilibili"
@@ -734,7 +736,8 @@ def download_playlist_by_url(self, url, **kwargs):
             api_url = self.bilibili_space_video_api(mid)
             api_content = get_content(api_url, headers=self.bilibili_headers())
             videos_info = json.loads(api_content)
-            pc = videos_info['data']['page']['count'] // videos_info['data']['page']['ps']
+            # pc = videos_info['data']['page']['count'] // videos_info['data']['page']['ps']
+            pc = math.ceil(videos_info['data']['page']['count'] / videos_info['data']['page']['ps'])
 
             for pn in range(1, pc + 1):
                 api_url = self.bilibili_space_video_api(mid, pn=pn)

From 439354e730d8b864de9401536c93220467ccb355 Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Wed, 24 Mar 2021 17:48:59 +0000
Subject: [PATCH 1027/1225] add HDR support for bilibili

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a812d72df3..a696b398b5 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -12,6 +12,8 @@ class Bilibili(VideoExtractor):
 
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
+        {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '3840p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '2160p', 'desc': '超清 4K'},
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,

From 25204d8841cdbbcad4f5df357c80853f34286025 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Mar 2021 16:38:54 +0200
Subject: [PATCH 1028/1225] [test] remove test_bilibili (videos deleted)

---
 tests/test.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 0f7595b316..5a86ee8fa4 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,14 +40,6 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_bilibili(self):
-        bilibili.download(
-            "https://www.bilibili.com/watchlater/#/BV1PE411q7mZ/p6", info_only=True
-        )
-        bilibili.download(
-            "https://www.bilibili.com/watchlater/#/av74906671/p6", info_only=True
-        )
-
     def test_soundcloud(self):
         ## single song
         soundcloud.download(

From 17eff492fe1db0c378f2447b54c5f09ed98b2626 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 29 Mar 2021 16:44:07 +0200
Subject: [PATCH 1029/1225] version 0.4.1520

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 95505814c5..60bff60740 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1500'
+__version__ = '0.4.1520'

From ef9ff72183acd93b1b10b2b836d145447cceb016 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 2 Apr 2021 01:44:36 +0200
Subject: [PATCH 1030/1225] [bilibili] fix a bug for non-interative multi-part
 videos
 (https://github.com/soimort/you-get/pull/2746#pullrequestreview-626492105)

---
 src/you_get/extractors/bilibili.py | 32 ++++++++++--------------------
 1 file changed, 11 insertions(+), 21 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index a696b398b5..644c5af4f5 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -181,7 +181,7 @@ def prepare(self, **kwargs):
             self.download_playlist_by_url(self.url, **kwargs)
             return
 
-        # regular av video
+        # regular video
         if sort == 'video':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
@@ -601,13 +601,21 @@ def download_playlist_by_url(self, url, **kwargs):
             log.e('[Error] Unsupported URL pattern.')
             exit(1)
 
-        # regular av video
+        # regular video
         if sort == 'video':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME
             initial_state = json.loads(initial_state_text)
             aid = initial_state['videoData']['aid']
             pn = initial_state['videoData']['videos']
-            if pn!= len(initial_state['videoData']['pages']):#interaction video 互动视频
+
+            if pn == len(initial_state['videoData']['pages']):
+                # non-interative video
+                for pi in range(1, pn + 1):
+                     purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi)
+                     self.__class__().download_by_url(purl, **kwargs)
+
+            else:
+                # interative video
                 search_node_list = []
                 download_cid_set = set([initial_state['videoData']['cid']])
                 params = {
@@ -658,24 +666,6 @@ def download_playlist_by_url(self, url, **kwargs):
                                     self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
                                 self.extract(**kwargs)
                                 self.download(**kwargs)
-            else:
-                playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo = json.loads(playinfo_text) if playinfo_text else None
-
-                html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
-                playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
-                playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-                p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or '1')-1
-                for pi in range(p,pn):
-                    self.prepare_by_cid(aid,initial_state['videoData']['pages'][pi]['cid'],'%s (P%s. %s)' % (initial_state['videoData']['title'], pi+1, initial_state['videoData']['pages'][pi]['part']),html_content,playinfo,playinfo_,url)
-                    try:
-                        self.streams_sorted = [dict([('id', stream_type['id'])] + list(self.streams[stream_type['id']].items())) for stream_type in self.__class__.stream_types if stream_type['id'] in self.streams]
-                    except:
-                        self.streams_sorted = [dict([('itag', stream_type['itag'])] + list(self.streams[stream_type['itag']].items())) for stream_type in self.__class__.stream_types if stream_type['itag'] in self.streams]
-                    self.extract(**kwargs)
-                    self.download(**kwargs)
-                    # purl = 'https://www.bilibili.com/video/av%s?p=%s' % (aid, pi+1)
-                    # self.__class__().download_by_url(purl, **kwargs)
 
         elif sort == 'bangumi':
             initial_state_text = match1(html_content, r'__INITIAL_STATE__=(.*?);\(function\(\)')  # FIXME

From 1c841f7e8ce60130572a8f03fb038eda99deff6a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 4 Apr 2021 02:59:00 +0200
Subject: [PATCH 1031/1225] [bilibili] redirect /s/ URLs

---
 src/you_get/extractors/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 644c5af4f5..edb656c706 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -162,6 +162,11 @@ def prepare(self, **kwargs):
             self.url = 'https://www.bilibili.com/bangumi/play/ep%s' % ep_id
             html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))
 
+        # redirect: s
+        elif re.match(r'https?://(www\.)?bilibili\.com/s/(.+)', self.url):
+            self.url = 'https://www.bilibili.com/%s' % match1(self.url, r'/s/(.+)')
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
         # sort it out
         if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
             sort = 'audio'

From 144886840212d5d0ee059858e6493dd265927376 Mon Sep 17 00:00:00 2001
From: flewsea <w48325832@gmail.com>
Date: Fri, 7 May 2021 00:20:30 +0800
Subject: [PATCH 1032/1225] skip private video

---
 src/you_get/extractors/iwara.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/iwara.py b/src/you_get/extractors/iwara.py
index 67a41d414c..37cd712a98 100644
--- a/src/you_get/extractors/iwara.py
+++ b/src/you_get/extractors/iwara.py
@@ -27,6 +27,9 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     api_url = video_url + '/api/video/' + video_hash
     content = get_content(api_url, headers=headers)
     data = json.loads(content)
+    if len(data)<1 :
+        print('Maybe is Private Video?'+'['+title+']')
+        return True;
     down_urls = 'https:' + data[0]['uri']
     type, ext, size = url_info(down_urls, headers=headers)
     print_info(site_info, title+data[0]['resolution'], type, size)
@@ -35,10 +38,8 @@ def iwara_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         download_urls([down_urls], title, ext, size, output_dir, merge=merge, headers=headers)
 
 def download_playlist_by_url( url, **kwargs):
-    video_page = get_content(url)
-    # url_first=re.findall(r"(http[s]?://[^/]+)",url)
+    video_page = get_html(url)
     url_first=match1(url, r"(http[s]?://[^/]+)")
-    # print (url_first)
     videos = set(re.findall(r'<a href="(/videos/[^"]+)"', video_page))
     if(len(videos)>0):
         for video in videos:

From 663e53a95f5435afd95a03bb8b16c6d1fac283a5 Mon Sep 17 00:00:00 2001
From: zhouyuan1 <zhouyuan1@staff.weibo.com>
Date: Thu, 20 May 2021 13:03:03 +0800
Subject: [PATCH 1033/1225] add param m3u8 , allow download video via m3u8 url

---
 src/you_get/common.py | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 67ef581b1b..6e619c1147 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1321,7 +1321,13 @@ def download_main(download, download_playlist, urls, playlist, **kwargs):
         if re.match(r'https?://', url) is None:
             url = 'http://' + url
 
-        if playlist:
+        if m3u8:
+            if output_filename:
+                title = output_filename
+            else:
+                title = "m3u8file"
+            download_url_ffmpeg(url=url, title=title,ext = 'mp4',output_dir = '.')
+        elif playlist:
             download_playlist(url, **kwargs)
         else:
             download(url, **kwargs)
@@ -1425,7 +1431,6 @@ def set_socks_proxy(proxy):
             proxy_info = proxy.split("@")
             socks_proxy_addrs = proxy_info[1].split(':')
             socks_proxy_auth = proxy_info[0].split(":")
-            print(socks_proxy_auth[0]+" "+socks_proxy_auth[1]+" "+socks_proxy_addrs[0]+" "+socks_proxy_addrs[1])
             socks.set_default_proxy(
                 socks.SOCKS5,
                 socks_proxy_addrs[0],
@@ -1436,7 +1441,6 @@ def set_socks_proxy(proxy):
             )
         else:
            socks_proxy_addrs = proxy.split(':')
-           print(socks_proxy_addrs[0]+" "+socks_proxy_addrs[1])
            socks.set_default_proxy(
                socks.SOCKS5,
                socks_proxy_addrs[0],
@@ -1601,6 +1605,10 @@ def print_version():
     download_grp.add_argument('--stream', help=argparse.SUPPRESS)
     download_grp.add_argument('--itag', help=argparse.SUPPRESS)
 
+    download_grp.add_argument('-m', '--m3u8', action='store_true', default=False,
+        help = 'download vide using an m3u8 url')
+
+
     parser.add_argument('URL', nargs='*', help=argparse.SUPPRESS)
 
     args = parser.parse_args()
@@ -1626,6 +1634,7 @@ def print_version():
     global output_filename
     global auto_rename
     global insecure
+    global m3u8
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
@@ -1647,6 +1656,9 @@ def print_version():
     if args.cookies:
         load_cookies(args.cookies)
 
+    if args.m3u8:
+        m3u8 = True
+
     caption = True
     stream_id = args.format or args.stream or args.itag
     if args.no_caption:

From 5104dd3058c342998b32dabdd47230c5f3af75c6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:38:02 +0200
Subject: [PATCH 1034/1225] [youtube] close #2890

---
 src/you_get/extractors/youtube.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 7e4c74f071..487869cfb6 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -195,8 +195,9 @@ def prepare(self, **kwargs):
         # Get video info
         # 'eurl' is a magic parameter that can bypass age restriction
         # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
-        video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
-        logging.debug('STATUS: %s' % video_info['status'][0])
+        #video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
+        #logging.debug('STATUS: %s' % video_info['status'][0])
+        video_info = {'status': ['ok'], 'use_cipher_signature': 'True'}
 
         ytplayer_config = None
         if 'status' not in video_info:
@@ -253,11 +254,16 @@ def prepare(self, **kwargs):
             else:
                 # Parse video page instead
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-                ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
 
-                self.title = json.loads(ytplayer_config["args"]["player_response"])["videoDetails"]["title"]
-                self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
-                stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
+                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+
+                self.title = ytInitialPlayerResponse["videoDetails"]["title"]
+                if re.search('([^"]*/base\.js)"', video_page):
+                    self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
+                else:
+                    self.html5player = None
+
+                stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         elif video_info['status'] == ['fail']:
             logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])

From e1db00a8fdc228f695b7c643823303ab7e3fe0f0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:38:53 +0200
Subject: [PATCH 1035/1225] test on Python 3.9

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b3d50ff741..daae6668b0 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, pypy3]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, pypy3]
 
     steps:
     - uses: actions/checkout@v2

From f9fbe8fae02a6ada9f64e0dcfbb960c856701ad6 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 24 May 2021 15:43:24 +0200
Subject: [PATCH 1036/1225] version 0.4.1525

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 60bff60740..63b4af5601 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1520'
+__version__ = '0.4.1525'

From 0fcbe3c5a79142f141a55cdd6705f562384b0eed Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 1 Jun 2021 18:14:56 +0200
Subject: [PATCH 1037/1225] [youtube] s_to_sig: add prefix to prevent namespace
 pollution

---
 src/you_get/extractors/youtube.py | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 487869cfb6..58614c5f99 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -76,11 +76,13 @@ def s_to_sig(js, s):
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl_RGK2l/en_US/base.js
         # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
+        # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
-            code = re.sub(r'(\W)(as|if|in|is|or)\(', r'\1_\2(', code)
+            # add prefix '_sig_' to prevent namespace pollution
+            code = re.sub(r'(\W)([$\w][$\w])\(', r'\1_sig_\2(', code)
             code = re.sub(r'\$', '_dollar', code)
-            code = re.sub(r'\{', r':\n\t', code)
+            code = re.sub(r'\{', r': ', code)
             code = re.sub(r'\}', r'\n', code)
             code = re.sub(r'var\s+', r'', code)
             code = re.sub(r'(\w+).join\(""\)', r'"".join(\1)', code)
@@ -99,7 +101,7 @@ def tr_js(code):
         f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
                 match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
         f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
-        f1def = 'function main_%s%s' % (f1, f1def)  # prefix to avoid potential namespace conflict
+        f1def = 'function %s%s' % (f1, f1def)
         code = tr_js(f1def)
         f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
         for f2 in f2s:
@@ -112,13 +114,13 @@ def tr_js(code):
                 f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
             f2 = re.sub(r'(as|if|in|is|or)', r'_\1', f2)
             f2 = re.sub(r'\$', '_dollar', f2)
-            code = code + 'global %s\n' % f2 + tr_js(f2def)
+            code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
 
         f1 = re.sub(r'(as|if|in|is|or)', r'_\1', f1)
         f1 = re.sub(r'\$', '_dollar', f1)
-        code = code + 'sig=main_%s(s)' % f1  # prefix to avoid potential namespace conflict
+        code = code + '_sig=_sig_%s(s)' % f1
         exec(code, globals(), locals())
-        return locals()['sig']
+        return locals()['_sig']
 
     def chunk_by_range(url, size):
         urls = []

From edb358d0b878ba12824b4a39dd5f28cab41a051f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 1 Jun 2021 18:33:26 +0200
Subject: [PATCH 1038/1225] version 0.4.1527

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 63b4af5601..70ca2ef5f7 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1525'
+__version__ = '0.4.1527'

From d40a15de5c15babe7daa4eb939e5c2f50a90f68c Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Fri, 4 Jun 2021 22:35:20 +0800
Subject: [PATCH 1039/1225] fix: url

---
 src/you_get/extractors/missevan.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index c2b25a90d9..5018852f2f 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -353,7 +353,7 @@ def url_danmaku_api(sound_id):
 
     @staticmethod
     def url_resource(uri):
-        return 'https://static.missevan.com/' + uri
+        return uri if re.match(r'^https?:/{2}\w.+$', uri) else 'https://static.missevan.com/' + uri
 
 site = MissEvan()
 site_info = 'MissEvan.com'

From 3881ed3f949ee63d53fb8f7f7b1e28c4f654ab25 Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Fri, 4 Jun 2021 22:35:50 +0800
Subject: [PATCH 1040/1225] feat: new missevan_stream_types

---
 src/you_get/extractors/missevan.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 5018852f2f..b7f413f20a 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -75,17 +75,13 @@ def dispatch(self, url, *args, **kwargs):
         raise _NoMatchException()
 
 missevan_stream_types = [
-    {'id': 'source', 'quality': '源文件', 'url_json_key': 'soundurl',
-     'resource_url_fmt': 'sound/{resource_url}'},
-    {'id': '320', 'quality': '320 Kbps', 'url_json_key': 'soundurl_64'},
+    {'id': 'source', 'quality': '源文件', 'url_json_key': 'soundurl'},
     {'id': '128', 'quality': '128 Kbps', 'url_json_key': 'soundurl_128'},
-    {'id': '32', 'quality': '32 Kbps', 'url_json_key': 'soundurl_32'},
     {'id': 'covers', 'desc': '封面图', 'url_json_key': 'cover_image',
      'default_src': 'covers/nocover.png',
      'resource_url_fmt': 'covers/{resource_url}'},
-    {'id': 'coversmini', 'desc': '封面缩略图', 'url_json_key': 'cover_image',
-     'default_src': 'coversmini/nocover.png',
-     'resource_url_fmt': 'coversmini/{resource_url}'}
+    {'id': 'coversmini', 'desc': '封面缩略图', 'url_json_key': 'front_cover',
+     'default_src': 'coversmini/nocover.png'}
 ]
 
 def _get_resource_uri(data, stream_type):

From 5445f5ecde1823a200ec24ea0b74328b95299b57 Mon Sep 17 00:00:00 2001
From: Philip Xu <pyx@xrefactor.com>
Date: Tue, 22 Jun 2021 17:51:17 -0400
Subject: [PATCH 1041/1225] Update douyin.py

The site douyin.com changed to a more user friendly website, this is the updated extractor for that.
---
 src/you_get/extractors/douyin.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index e39775f44f..5de1fdd912 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -1,7 +1,7 @@
 # coding=utf-8
 
 import re
-import json
+from urllib.parse import unquote
 
 from ..common import (
     url_size,
@@ -18,17 +18,17 @@
 
 def douyin_download_by_url(url, **kwargs):
     page_content = get_content(url, headers=fake_headers)
-    match_rule = re.compile(r'var data = \[(.*?)\];')
-    video_info = json.loads(match_rule.findall(page_content)[0])
-    video_url = video_info['video']['play_addr']['url_list'][0]
-    # fix: https://www.douyin.com/share/video/6553248251821165832
-    # if there is no title, use desc
-    cha_list = video_info['cha_list']
-    if cha_list:
-        title = cha_list[0]['cha_name']
-    else:
-        title = video_info['desc']
+    # The easiest way to get the title is, obviously, from <title>
+    title = re.findall(r'<title.*>(.*)</title>', page_content)[0].strip()
+    # Remove the site name from title
+    site_name = ' - 抖音'
+    if title.endswith(site_name):
+        title = title[:-len(site_name)]
     video_format = 'mp4'
+    # The video url is url escaped, as of today, there are 4 working CDN video
+    # urls for the same video, I chose the shortest one.
+    cdn_pattern = r'(api\.amemv\.com.*PackSourceEnum_AWEME_DETAIL)'
+    video_url = 'https://' + unquote(re.findall(cdn_pattern, page_content)[0])
     size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,

From b9dbae8b40588bc134af6bb7406492ec68fffded Mon Sep 17 00:00:00 2001
From: zhancat200801 <zhancat200801@sina.com>
Date: Tue, 29 Jun 2021 13:18:09 +0800
Subject: [PATCH 1042/1225] modify qq.py

---
 src/you_get/extractors/qq.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index 6411b19552..e38770e9c0 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -35,6 +35,7 @@ def qq_download_by_vid(vid, title, output_dir='.', merge=True, info_only=False):
 
     part_urls= []
     total_size = 0
+    ext = None
     for part in range(1, seg_cnt+1):
         if fc_cnt == 0:
             # fix json parsing error

From 71780ae4aa4a47fd817dc4c8485ce4aac96a8633 Mon Sep 17 00:00:00 2001
From: Philip Xu <pyx@xrefactor.com>
Date: Tue, 6 Jul 2021 06:24:52 -0400
Subject: [PATCH 1043/1225] Update douyin.py

Updated the extractors for the latest www.douyin.com
---
 src/you_get/extractors/douyin.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 5de1fdd912..8067b1b564 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -1,6 +1,7 @@
 # coding=utf-8
 
 import re
+import json
 from urllib.parse import unquote
 
 from ..common import (
@@ -18,17 +19,17 @@
 
 def douyin_download_by_url(url, **kwargs):
     page_content = get_content(url, headers=fake_headers)
-    # The easiest way to get the title is, obviously, from <title>
-    title = re.findall(r'<title.*>(.*)</title>', page_content)[0].strip()
-    # Remove the site name from title
-    site_name = ' - 抖音'
-    if title.endswith(site_name):
-        title = title[:-len(site_name)]
+    # The video player and video source are rendered client-side, the data
+    # contains in a <script id="RENDER_DATA" type="application/json"> tag
+    # quoted, unquote the whole page content then search using regex with
+    # regular string.
+    page_content = unquote(page_content)
+    title = re.findall(r'"desc":"([^"]*)"', page_content)[0].strip()
     video_format = 'mp4'
-    # The video url is url escaped, as of today, there are 4 working CDN video
-    # urls for the same video, I chose the shortest one.
-    cdn_pattern = r'(api\.amemv\.com.*PackSourceEnum_AWEME_DETAIL)'
-    video_url = 'https://' + unquote(re.findall(cdn_pattern, page_content)[0])
+    # video URLs are in this pattern {"src":"THE_URL"}, in json format
+    urls_pattern = r'"playAddr":(\[.*?\])'
+    urls = json.loads(re.findall(urls_pattern, page_content)[0])
+    video_url = 'https:' + urls[0]['src']
     size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,

From 69c61b11760e1650b071b8273349f07df482edee Mon Sep 17 00:00:00 2001
From: kagamia <amethyst50504724@msn.com>
Date: Sun, 11 Jul 2021 21:49:31 +0800
Subject: [PATCH 1044/1225] miaopai: use weibo h5 api

---
 src/you_get/extractors/miaopai.py | 51 ++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 0ddcadba0b..05c1e650b3 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -19,7 +19,7 @@
 
 def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Source: Android mobile'''
-    page_url = 'http://video.weibo.com/show?fid=' + fid + '&type=mp4'
+    page_url = 'https://video.weibo.com/show?fid=' + fid + '&type=mp4'
 
     mobile_page = get_content(page_url, headers=fake_headers_mobile)
     url = match1(mobile_page, r'<video id=.*?src=[\'"](.*?)[\'"]\W')
@@ -78,6 +78,51 @@ def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **
         download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
 
 
+def miaopai_download_h5api(url, output_dir='.', merge=False, info_only=False, **kwargs):
+    oid = match1(url, r'/show/(\d{4}:\w+)')
+    page = "/show/%s" % oid
+    data_url = 'https://h5.video.weibo.com/api/component?%s' % parse.urlencode({
+        'page': page
+    })
+    headers = {}
+    headers.update(fake_headers_mobile)
+    headers['origin'] = 'https://h5.video.weibo.com'
+    headers['page-referer'] = page
+    headers['referer'] = 'https://h5.video.weibo.com/show/%s' % oid
+    post_data = {
+        "data": json.dumps({
+            "Component_Play_Playinfo": {"oid": oid}
+        })
+    }
+    data_content = post_content(data_url, headers=headers, post_data=post_data)
+    data = json.loads(data_content)
+    if data['msg'] != 'succ':
+        raise Exception('Weibo api returns non-success: (%s)%s'.format(data['code'], data['msg']))
+
+    play_info = data['data']['Component_Play_Playinfo']
+    title = play_info['title']
+
+    # get video formats and sort by size desc
+    video_formats = []
+    for fmt, relative_uri in play_info['urls'].items():
+        url = "https:%s" % relative_uri
+        type, ext, size = url_info(url, headers=headers)
+        video_formats.append({
+            'fmt': fmt,
+            'url': url,
+            'type': type,
+            'ext': ext,
+            'size': size,
+        })
+    video_formats.sort(key=lambda v:v['size'], reverse=True)
+    selected_video = video_formats[0]
+    video_url, ext, size = selected_video['url'], selected_video['ext'], selected_video['size']
+
+    print_info(site_info, title, ext, size)
+    if not info_only:
+        download_urls([video_url], fs.legitimize(title), ext, total_size=size, headers=headers, **kwargs)
+
+
 def miaopai_download_direct(url, output_dir='.', merge=False, info_only=False, **kwargs):
     mobile_page = get_content(url, headers=fake_headers_mobile)
     try:
@@ -108,12 +153,16 @@ def miaopai_download(url, output_dir='.', merge=False, info_only=False, **kwargs
     if re.match(r'^http[s]://.*\.weibo\.com/tv/v/(\w+)', url):
         return miaopai_download_direct(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
+    if re.match(r'^http[s]://(.+\.)?weibo\.com/(tv/)?show/(\d{4}:\w+)', url):
+        return miaopai_download_h5api(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)
     elif '/p/230444' in url:
         fid = match1(url, r'/p/230444(\w+)')
         miaopai_download_by_fid('1034:'+fid, output_dir, merge, info_only)
+        pass
     else:
         mobile_page = get_content(url, headers = fake_headers_mobile)
         hit = re.search(r'"page_url"\s*:\s*"([^"]+)"', mobile_page)

From 3044f5c16aa4742bd71127bff5211273d638156a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:34:33 +0200
Subject: [PATCH 1045/1225] [common] manually add cookies in get_content and
 post_content (due to python's lack of support for #HttpOnly_)

---
 src/you_get/common.py | 26 ++++++++++++++++++++++----
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 67ef581b1b..597ed45af8 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -433,8 +433,17 @@ def get_content(url, headers={}, decoded=True):
 
     req = request.Request(url, headers=headers)
     if cookies:
-        cookies.add_cookie_header(req)
-        req.headers.update(req.unredirected_hdrs)
+        # NOTE: Do not use cookies.add_cookie_header(req)
+        # #HttpOnly_ cookies were not supported by CookieJar and MozillaCookieJar properly until python 3.10
+        # See also:
+        # - https://github.com/python/cpython/pull/17471
+        # - https://bugs.python.org/issue2190
+        # Here we add cookies to the request headers manually
+        cookie_strings = []
+        for cookie in list(cookies):
+            cookie_strings.append(cookie.name + '=' + cookie.value)
+        cookie_headers = {'Cookie': '; '.join(cookie_strings)}
+        req.headers.update(cookie_headers)
 
     response = urlopen_with_retry(req)
     data = response.read()
@@ -477,8 +486,17 @@ def post_content(url, headers={}, post_data={}, decoded=True, **kwargs):
 
     req = request.Request(url, headers=headers)
     if cookies:
-        cookies.add_cookie_header(req)
-        req.headers.update(req.unredirected_hdrs)
+        # NOTE: Do not use cookies.add_cookie_header(req)
+        # #HttpOnly_ cookies were not supported by CookieJar and MozillaCookieJar properly until python 3.10
+        # See also:
+        # - https://github.com/python/cpython/pull/17471
+        # - https://bugs.python.org/issue2190
+        # Here we add cookies to the request headers manually
+        cookie_strings = []
+        for cookie in list(cookies):
+            cookie_strings.append(cookie.name + '=' + cookie.value)
+        cookie_headers = {'Cookie': '; '.join(cookie_strings)}
+        req.headers.update(cookie_headers)
     if kwargs.get('post_data_raw'):
         post_data_enc = bytes(kwargs['post_data_raw'], 'utf-8')
     else:

From 47bd107686802c2ee0c3faa083c39d07b7db59dd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:41:05 +0200
Subject: [PATCH 1046/1225] [instagram] use get_content

---
 src/you_get/extractors/instagram.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 0a9da3458f..e7c7971714 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -6,14 +6,14 @@
 
 def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = r1(r'([^?]*)', url)
-    html = get_html(url, faker=True)
+    cont = get_content(url, headers=fake_headers)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
-    description = r1(r'<meta property="og:title" content="([^"]*)"', html) or \
-        r1(r'<title>\s([^<]*)</title>', html) # with logged-in cookies
+    description = r1(r'<meta property="og:title" content="([^"]*)"', cont) or \
+        r1(r'<title>\s([^<]*)</title>', cont) # with logged-in cookies
     title = "{} [{}]".format(description.replace("\n", " "), vid)
 
-    stream = r1(r'<meta property="og:video" content="([^"]*)"', html)
+    stream = r1(r'<meta property="og:video" content="([^"]*)"', cont)
     if stream:
         _, ext, size = url_info(stream)
 
@@ -21,14 +21,14 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         if not info_only:
             download_urls([stream], title, ext, size, output_dir, merge=merge)
     else:
-        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', html)
+        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', cont)
         try:
             info = json.loads(data.group(1))
             post = info['entry_data']['PostPage'][0]
             assert post
         except:
             # with logged-in cookies
-            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', html)
+            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', cont)
             if data is not None:
                 log.e('[Warning] Cookies needed.')
             post = json.loads(data.group(1))

From ad24e68baa5106522c6d4a37395896e9776ea88f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Jul 2021 18:46:41 +0200
Subject: [PATCH 1047/1225] version 0.4.1536

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 70ca2ef5f7..dd60391801 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1527'
+__version__ = '0.4.1536'

From f4ec55b00e5b8743727b138d06f6ffadadd49579 Mon Sep 17 00:00:00 2001
From: Tim Gates <tim.gates@iress.com>
Date: Sat, 31 Jul 2021 09:34:46 +1000
Subject: [PATCH 1048/1225] docs: Fix a few typos

There are small typos in:
- src/you_get/extractors/flickr.py
- src/you_get/extractors/mtv81.py
- src/you_get/extractors/qingting.py

Fixes:
- Should read `several` rather than `serveral`.
- Should read `channel` rather than `chaanel`.
- Should read `approach` rather than `approch`.
---
 src/you_get/extractors/flickr.py   | 2 +-
 src/you_get/extractors/mtv81.py    | 2 +-
 src/you_get/extractors/qingting.py | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/flickr.py b/src/you_get/extractors/flickr.py
index 2535dd1cb7..79fca4ff9a 100644
--- a/src/you_get/extractors/flickr.py
+++ b/src/you_get/extractors/flickr.py
@@ -73,7 +73,7 @@ def get_api_key(page):
     match = match1(page, pattern_inline_api_key)
     # this happens only when the url points to a gallery page
     # that contains no inline api_key(and never makes xhr api calls)
-    # in fact this might be a better approch for getting a temporary api key
+    # in fact this might be a better approach for getting a temporary api key
     # since there's no place for a user to add custom information that may
     # misguide the regex in the homepage
     if not match:
diff --git a/src/you_get/extractors/mtv81.py b/src/you_get/extractors/mtv81.py
index b92f74bc2d..ef43215959 100644
--- a/src/you_get/extractors/mtv81.py
+++ b/src/you_get/extractors/mtv81.py
@@ -28,7 +28,7 @@ def mtv81_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     #
     # rtmpdump  -r 'rtmpe://cp30865.edgefcs.net/ondemand/mtviestor/_!/intlod/MTVInternational/MBUS/GeoLocals/00JP/VIAMTVI/PYC/201304/7122HVAQ4/00JPVIAMTVIPYC7122HVAQ4_640x_360_1200_m30.mp4' -o "title.mp4" --swfVfy http://media.mtvnservices.com/player/prime/mediaplayerprime.1.10.8.swf
     #
-    # because rtmpdump is unstable,may try serveral times
+    # because rtmpdump is unstable,may try several times
     #
     if not info_only:
         # import pdb
diff --git a/src/you_get/extractors/qingting.py b/src/you_get/extractors/qingting.py
index 9859d4be95..8dd1b14f56 100644
--- a/src/you_get/extractors/qingting.py
+++ b/src/you_get/extractors/qingting.py
@@ -10,7 +10,7 @@
 
 class Qingting(VideoExtractor):
     # every resource is described by its channel id and program id
-    # so vid is tuple (chaanel_id, program_id)
+    # so vid is tuple (channel_id, program_id)
 
     name = 'Qingting'
     stream_types = [

From b97e9484430e2344d3c332d39880ba3ae68890e8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 05:54:36 +0200
Subject: [PATCH 1049/1225] [youtube] tr_js: support 3-char main function names

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 58614c5f99..81b45ac5b5 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -77,10 +77,11 @@ def s_to_sig(js, s):
         # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
         # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
+        # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             # add prefix '_sig_' to prevent namespace pollution
-            code = re.sub(r'(\W)([$\w][$\w])\(', r'\1_sig_\2(', code)
+            code = re.sub(r'(\W)([$\w][$\w][$\w]?)\(', r'\1_sig_\2(', code)
             code = re.sub(r'\$', '_dollar', code)
             code = re.sub(r'\{', r': ', code)
             code = re.sub(r'\}', r'\n', code)

From 09cd505311af7dff87d344436a7f4a87b1763cd1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:39:48 +0200
Subject: [PATCH 1050/1225] [test] disable test_soundcloud temporarily

---
 tests/test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 5a86ee8fa4..1989455f69 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -42,9 +42,9 @@ def test_acfun(self):
 
     def test_soundcloud(self):
         ## single song
-        soundcloud.download(
-            'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
-        )
+        #soundcloud.download(
+        #    'https://soundcloud.com/keiny-pham/impure-bird', info_only=True
+        #)
         ## playlist
         #soundcloud.download(
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True

From e1e1503b08c037ec0eb06c6c951240d004cf342b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:44:15 +0200
Subject: [PATCH 1051/1225] [test] disable test_soundcloud temporarily

---
 tests/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index 1989455f69..4a2a117c7f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,7 +40,7 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_soundcloud(self):
+    #def test_soundcloud(self):
         ## single song
         #soundcloud.download(
         #    'https://soundcloud.com/keiny-pham/impure-bird', info_only=True

From 5498c377ff3e8c03831ecef9defe18bea8b4937d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 Aug 2021 06:47:09 +0200
Subject: [PATCH 1052/1225] version 0.4.1545

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index dd60391801..f7daa7f8b3 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1536'
+__version__ = '0.4.1545'

From ea5f712cb0a420e26ea6e883014159deb584c43b Mon Sep 17 00:00:00 2001
From: hong56hk <hong56hk@gmail.com>
Date: Thu, 26 Aug 2021 00:12:52 +0800
Subject: [PATCH 1053/1225] fix for using insecure flag does not work

---
 src/you_get/common.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 597ed45af8..5aa74a20ba 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -343,7 +343,12 @@ def undeflate(data):
 # DEPRECATED in favor of get_content()
 def get_response(url, faker=False):
     logging.debug('get_response: %s' % url)
-
+    ctx = None
+    if insecure:
+        # ignore ssl errors
+        ctx = ssl.create_default_context()
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
     # install cookies
     if cookies:
         opener = request.build_opener(request.HTTPCookieProcessor(cookies))
@@ -351,10 +356,10 @@ def get_response(url, faker=False):
 
     if faker:
         response = request.urlopen(
-            request.Request(url, headers=fake_headers), None
+            request.Request(url, headers=fake_headers), None, context=ctx,
         )
     else:
-        response = request.urlopen(url)
+        response = request.urlopen(url, context=ctx)
 
     data = response.read()
     if response.info().get('Content-Encoding') == 'gzip':

From 63fd9716a8740fc6862b70a474e398ca6e9f26bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 27 Aug 2021 05:14:00 +0200
Subject: [PATCH 1054/1225] [universal] fix blogger

---
 src/you_get/extractors/universal.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index abc6947599..fdc7426dac 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -70,12 +70,13 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                       '[-_][6-9]\d\dx1\d\d\d\.jpe?g',
                       '[-_][6-9]\d\dx[6-9]\d\d\.jpe?g',
                       's1600/[\w%]+\.jpe?g', # blogger
+                      'blogger\.googleusercontent\.com/img/a/\w*', # blogger
                       'img[6-9]\d\d/[\w%]+\.jpe?g' # oricon?
         ]
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^ ;&"\'\\<>]+' + i + r'[^ ;&"\'\\<>]*)', page)
+            urls += re.findall(r'(https?://[^ ;&"\'\\<>]*' + i + r'[^ ;&"\'\\<>]*)', page)
 
             p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page)
             urls += [parse.unquote(url) for url in p_urls]

From d896342862b2fc18448fe2b21054af8ba27f8dbc Mon Sep 17 00:00:00 2001
From: Ziding Zhang <zidingz@gmail.com>
Date: Mon, 20 Sep 2021 14:41:37 +0100
Subject: [PATCH 1055/1225] Create SECURITY.md

I'd like to report a security issue but cannot find contact instructions on your repository.

If not a hassle, might you kindly add a SECURITY.md file with an email, or another contact method? GitHub [recommends](https://docs.github.com/en/code-security/getting-started/adding-a-security-policy-to-your-repository) this best practice to ensure security issues are responsibly disclosed, and it would serve as a simple instruction for security researchers in the future.

Thank you for your consideration!
---
 SECURITY.md | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 SECURITY.md

diff --git a/SECURITY.md b/SECURITY.md
new file mode 100644
index 0000000000..5041b2f202
--- /dev/null
+++ b/SECURITY.md
@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Please report security issues to <email>

From 7c2523f5cc843a6622968f0eed8dcb9a49b62b00 Mon Sep 17 00:00:00 2001
From: Jian Wang <wantee.wang@gmail.com>
Date: Thu, 7 Oct 2021 15:42:37 +0800
Subject: [PATCH 1056/1225] support channel/series for bilibili

---
 src/you_get/extractors/bilibili.py | 20 ++++++++++++++++++++
 tests/test.py                      |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index edb656c706..36de363c04 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -112,6 +112,10 @@ def bilibili_live_room_init_api(room_id):
     def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
 
+    @staticmethod
+    def bilibili_series_archives_api(mid, sid, pn=1, ps=100):
+        return 'https://api.bilibili.com/x/series/archives?mid=%s&series_id=%s&pn=%s&ps=%s&only_normal=true&sort=asc&jsonp=jsonp' % (mid, sid, pn, ps)
+
     @staticmethod
     def bilibili_space_favlist_api(fid, pn=1, ps=20):
         return 'https://api.bilibili.com/x/v3/fav/resource/list?media_id=%s&pn=%s&ps=%s&order=mtime&type=0&tid=0&jsonp=jsonp' % (fid, pn, ps)
@@ -596,6 +600,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'video'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
             sort = 'space_channel'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/seriesdetail\?.*sid=(\d+)', self.url):
+            sort = 'space_channel_series'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
             sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
@@ -706,6 +712,20 @@ def download_playlist_by_url(self, url, **kwargs):
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
 
+        elif sort == 'space_channel_series':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/seriesdetail\?.*sid=(\d+)', self.url)
+            mid, sid = m.group(1), m.group(2)
+            api_url = self.bilibili_series_archives_api(mid, sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            archives_info = json.loads(api_content)
+            # TBD: channel of more than 100 videos
+
+            epn, i = len(archives_info['data']['archives']), 0
+            for video in archives_info['data']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+
         elif sort == 'space_favlist':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
             vmid, fid = m.group(1), m.group(2)
diff --git a/tests/test.py b/tests/test.py
index 4a2a117c7f..8ae622b2b8 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,6 +40,9 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
+    def test_bilibili(self):
+        bilibili.download('https://space.bilibili.com/72270557/channel/seriesdetail?sid=218844', info_only=True)
+
     #def test_soundcloud(self):
         ## single song
         #soundcloud.download(

From 798ad6d14eb23ee0754ead224494911fcf27cd68 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 20 Oct 2021 18:15:51 +0200
Subject: [PATCH 1057/1225] [universal] tweak

---
 src/you_get/extractors/universal.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index fdc7426dac..4a3268ab89 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -76,7 +76,7 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         urls = []
         for i in media_exts:
-            urls += re.findall(r'(https?://[^ ;&"\'\\<>]*' + i + r'[^ ;&"\'\\<>]*)', page)
+            urls += re.findall(r'(https?://[^ ;&"\'\\<>]*' + i + r'[^ =?;&"\'\\<>]*)', page)
 
             p_urls = re.findall(r'(https?%3A%2F%2F[^;&"]+' + i + r'[^;&"]*)', page)
             urls += [parse.unquote(url) for url in p_urls]

From c064013b9c19c6225b6edd818ced0f4003d22854 Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Sun, 24 Oct 2021 07:58:34 +0200
Subject: [PATCH 1058/1225] Test on Python 3.10
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

“3.10” must be quoted or yaml will treat it as 3.1.
---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index daae6668b0..b23455c899 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, pypy3]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
 
     steps:
     - uses: actions/checkout@v2

From 3c8382d2af3ef9239f64f6524cbe1abeb949729c Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Sun, 24 Oct 2021 08:01:32 +0200
Subject: [PATCH 1059/1225] strategy: fail-fast: false

---
 .github/workflows/python-package.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index b23455c899..75231110ae 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -14,6 +14,7 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
 

From 968334acb48c79bc801ca61ee1d0f77d570ccadb Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Sun, 24 Oct 2021 08:06:43 +0200
Subject: [PATCH 1060/1225] "Programming Language :: Python :: 3.10",

---
 you-get.json | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/you-get.json b/you-get.json
index e98e2e8a4a..a9ef1dd594 100644
--- a/you-get.json
+++ b/you-get.json
@@ -25,6 +25,8 @@
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Multimedia",

From c4603bbd2a4377ea2ff6f9c16a280e8a01a8583c Mon Sep 17 00:00:00 2001
From: Chuang Zhu <chuang@melty.land>
Date: Tue, 2 Nov 2021 12:45:41 +0800
Subject: [PATCH 1061/1225] [bilibili] fix 'NoneType' object is not
 subscriptable

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index edb656c706..38ff368d06 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -193,10 +193,12 @@ def prepare(self, **kwargs):
 
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text) if playinfo_text else None
+            playinfo = playinfo if playinfo['code'] == 0 else None
 
             html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
             playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
+            playinfo_ = playinfo_ if playinfo_['code'] == 0 else None
 
             # warn if it is a multi-part video
             pn = initial_state['videoData']['videos']

From 8eb1cc3da01a43996cddafc2c19b9546eca40a33 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 3 Nov 2021 18:32:35 +0100
Subject: [PATCH 1062/1225] version 0.4.1555

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index f7daa7f8b3..e8b65aae76 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1545'
+__version__ = '0.4.1555'

From 437b0901337d2041f4ea58c9cd67c0518b5e1e91 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 4 Nov 2021 17:21:08 +0100
Subject: [PATCH 1063/1225] [test] reduce test_youtube

---
 tests/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 4a2a117c7f..f28a6c92cd 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -29,10 +29,10 @@ def test_youtube(self):
             'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True
         )
         youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True)
-        youtube.download(
-            'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
-            info_only=True
-        )
+        #youtube.download(
+        #    'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
+        #    info_only=True
+        #)
         #youtube.download(
         #    'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
         #)

From 424b555500e65da0533d369a80562bb93907ec9a Mon Sep 17 00:00:00 2001
From: Zhengyang Song <songzy_thu@163.com>
Date: Sun, 14 Nov 2021 23:00:32 +0800
Subject: [PATCH 1064/1225] Fix zhihu extractor comment: ld

---
 src/you_get/extractors/zhihu.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/zhihu.py b/src/you_get/extractors/zhihu.py
index 64f81423ca..1dceef5338 100644
--- a/src/you_get/extractors/zhihu.py
+++ b/src/you_get/extractors/zhihu.py
@@ -31,8 +31,8 @@ def zhihu_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
 
         play_list = video_info["playlist"]
         # first High Definition
-        # second Second Standard Definition
-        # third ld. What is ld ?
+        # second Standard Definition
+        # third Low Definition
         # finally continue
         data = play_list.get("hd", play_list.get("sd", play_list.get("ld", None)))
         if not data:

From 3f47a215bf5f73e73cd7aa607b340a7b68b9bd3d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Nov 2021 18:10:34 +0100
Subject: [PATCH 1065/1225] fix the "name m3u8 is not defined" bug of #2888

---
 src/you_get/common.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 529a6471c1..5786f1b6a4 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -136,6 +136,7 @@
 output_filename = None
 auto_rename = False
 insecure = False
+m3u8 = False
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa

From 5a12118c836b0b9a7a2f8bdbed25be9aa2ca7831 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Nov 2021 18:21:11 +0100
Subject: [PATCH 1066/1225] add param "--postfix" to postfix downloaded files
 with unique identifiers

---
 src/you_get/common.py    | 9 +++++++++
 src/you_get/extractor.py | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 5786f1b6a4..473c315592 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -137,6 +137,7 @@
 auto_rename = False
 insecure = False
 m3u8 = False
+postfix = False
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
@@ -984,6 +985,8 @@ def download_urls(
             pass
 
     title = tr(get_filename(title))
+    if postfix and 'vid' in kwargs:
+        title = "%s [%s]" % (title, kwargs['vid'])
     output_filename = get_output_filename(urls, title, ext, output_dir, merge)
     output_filepath = os.path.join(output_dir, output_filename)
 
@@ -1532,6 +1535,10 @@ def print_version():
         '--no-caption', action='store_true',
         help='Do not download captions (subtitles, lyrics, danmaku, ...)'
     )
+    download_grp.add_argument(
+        '--postfix', action='store_true', default=False,
+        help='Postfix downloaded files with unique identifiers'
+    )
     download_grp.add_argument(
         '-f', '--force', action='store_true', default=False,
         help='Force overwriting existing files'
@@ -1654,6 +1661,7 @@ def print_version():
     global auto_rename
     global insecure
     global m3u8
+    global postfix
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
@@ -1690,6 +1698,7 @@ def print_version():
         # ignore ssl
         insecure = True
 
+    postfix = args.postfix
 
     if args.no_proxy:
         set_http_proxy('')
diff --git a/src/you_get/extractor.py b/src/you_get/extractor.py
index c4315935e7..bd71717e72 100644
--- a/src/you_get/extractor.py
+++ b/src/you_get/extractor.py
@@ -238,7 +238,8 @@ def download(self, **kwargs):
             download_urls(urls, self.title, ext, total_size, headers=headers,
                           output_dir=kwargs['output_dir'],
                           merge=kwargs['merge'],
-                          av=stream_id in self.dash_streams)
+                          av=stream_id in self.dash_streams,
+                          vid=self.vid)
 
             if 'caption' not in kwargs or not kwargs['caption']:
                 print('Skipping captions or danmaku.')

From 884b56e5faa99353cd5499c27d43deb6184eba63 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 16 Dec 2021 16:35:58 +0100
Subject: [PATCH 1067/1225] [imgur] fix extraction of original image

---
 src/you_get/extractors/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 519fa245ca..3b1c0ea645 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -52,7 +52,7 @@ def prepare(self, **kwargs):
         else:
             # gallery image
             content = get_content(self.url)
-            url = match1(content, r'(https?://i.imgur.com/[^"]+)')
+            url = match1(content, r'meta property="og:image"[^>]+(https?://i.imgur.com/[^"?]+)')
             _, container, size = url_info(url)
             self.streams = {
                 'original': {

From d92e1aacd1dec1b0e712ee218de30037d8bcf0d7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Jan 2022 22:10:50 +0100
Subject: [PATCH 1068/1225] [imgur] prefer og:video

---
 src/you_get/extractors/imgur.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 3b1c0ea645..d612a30a88 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -52,7 +52,8 @@ def prepare(self, **kwargs):
         else:
             # gallery image
             content = get_content(self.url)
-            url = match1(content, r'meta property="og:image"[^>]+(https?://i.imgur.com/[^"?]+)')
+            url = match1(content, r'meta property="og:video"[^>]+(https?://i.imgur.com/[^"?]+)') or \
+                match1(content, r'meta property="og:image"[^>]+(https?://i.imgur.com/[^"?]+)')
             _, container, size = url_info(url)
             self.streams = {
                 'original': {

From 68cf21dbee1478ab86bd1608e84ceebc54e7a05f Mon Sep 17 00:00:00 2001
From: Peter <peter279k@gmail.com>
Date: Fri, 7 Jan 2022 10:41:05 +0800
Subject: [PATCH 1069/1225] Add pipenv command usage option

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index d2fdaa99c0..0c3d4099ab 100644
--- a/README.md
+++ b/README.md
@@ -89,6 +89,14 @@ $ python3 setup.py install --user
 
 to install `you-get` to a permanent path.
 
+You can also use the [pipenv](https://pipenv.pypa.io/en/latest) to install the `you-get` in the Python virtual environment.
+
+```
+$ pipenv install -e .
+$ pipenv run you-get --version
+you-get: version 0.4.1555, a tiny downloader that scrapes the web.
+```
+
 ### Option 4: Git clone
 
 This is the recommended way for all developers, even if you don't often code in Python.

From 25fa99c947771f78746a4e7a15014b4b074eae33 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 20 Jan 2022 02:41:32 +0100
Subject: [PATCH 1070/1225] [instagram] fix extraction

---
 src/you_get/extractors/instagram.py | 27 +++++----------------------
 1 file changed, 5 insertions(+), 22 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index e7c7971714..be503c87b4 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -33,13 +33,11 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 log.e('[Warning] Cookies needed.')
             post = json.loads(data.group(1))
 
-        if 'edge_sidecar_to_children' in post['graphql']['shortcode_media']:
-            edges = post['graphql']['shortcode_media']['edge_sidecar_to_children']['edges']
-            for edge in edges:
-                title = edge['node']['shortcode']
-                image_url = edge['node']['display_url']
-                if 'video_url' in edge['node']:
-                    image_url = edge['node']['video_url']
+        for item in post['items']:
+            code = item['code']
+            for i, media in enumerate(item['carousel_media']):
+                title = '%s [%s]' % (code, i)
+                image_url = media['image_versions2']['candidates'][0]['url']
                 ext = image_url.split('?')[0].split('.')[-1]
                 size = int(get_head(image_url)['Content-Length'])
 
@@ -50,21 +48,6 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                                   ext=ext,
                                   total_size=size,
                                   output_dir=output_dir)
-        else:
-            title = post['graphql']['shortcode_media']['shortcode']
-            image_url = post['graphql']['shortcode_media']['display_url']
-            if 'video_url' in post['graphql']['shortcode_media']:
-                image_url = post['graphql']['shortcode_media']['video_url']
-            ext = image_url.split('?')[0].split('.')[-1]
-            size = int(get_head(image_url)['Content-Length'])
-
-            print_info(site_info, title, ext, size)
-            if not info_only:
-                download_urls(urls=[image_url],
-                              title=title,
-                              ext=ext,
-                              total_size=size,
-                              output_dir=output_dir)
 
 site_info = "Instagram.com"
 download = instagram_download

From 582ce1d67837b2064af114beb065f01d263bfccf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 20 Jan 2022 03:00:57 +0100
Subject: [PATCH 1071/1225] [instagram] fix extraction

---
 src/you_get/extractors/instagram.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index be503c87b4..f22bc60dfc 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -35,7 +35,8 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
         for item in post['items']:
             code = item['code']
-            for i, media in enumerate(item['carousel_media']):
+            carousel_media = item.get('carousel_media') or [item]
+            for i, media in enumerate(carousel_media):
                 title = '%s [%s]' % (code, i)
                 image_url = media['image_versions2']['candidates'][0]['url']
                 ext = image_url.split('?')[0].split('.')[-1]

From 7017c9e73fb985735e64a9cac26562c663f90f93 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 21 Jan 2022 02:27:11 +0100
Subject: [PATCH 1072/1225] [instagram] fix video extraction

---
 src/you_get/extractors/instagram.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index f22bc60dfc..0ed31cbc84 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -50,6 +50,20 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                                   total_size=size,
                                   output_dir=output_dir)
 
+                # download videos (if any)
+                if 'video_versions' in media:
+                    video_url = media['video_versions'][0]['url']
+                    ext = video_url.split('?')[0].split('.')[-1]
+                    size = int(get_head(video_url)['Content-Length'])
+
+                    print_info(site_info, title, ext, size)
+                    if not info_only:
+                        download_urls(urls=[video_url],
+                                      title=title,
+                                      ext=ext,
+                                      total_size=size,
+                                      output_dir=output_dir)
+
 site_info = "Instagram.com"
 download = instagram_download
 download_playlist = playlist_not_supported('instagram')

From ed2c0b32fe430e1d8b7f927263015b86087a3fc2 Mon Sep 17 00:00:00 2001
From: SuperSodaSea <bobby825@126.com>
Date: Sat, 22 Jan 2022 22:50:21 +0800
Subject: [PATCH 1073/1225] [bilibili] Add 8K video download support

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5cd47e1050..00204da21d 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -12,6 +12,8 @@ class Bilibili(VideoExtractor):
 
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
+        {'id': 'hdflv2_8k', 'quality': 127, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '4320p', 'desc': '超高清 8K'},
         {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '3840p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,

From 5aeae31fa2553eedfdf11d1bbde7892184164f73 Mon Sep 17 00:00:00 2001
From: SuperSodaSea <bobby825@126.com>
Date: Sat, 29 Jan 2022 02:44:00 +0800
Subject: [PATCH 1074/1225] [bilibili] Add Dolby Vision video download support

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 00204da21d..4b2c246b4f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -14,6 +14,8 @@ class Bilibili(VideoExtractor):
     stream_types = [
         {'id': 'hdflv2_8k', 'quality': 127, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '4320p', 'desc': '超高清 8K'},
+        {'id': 'hdflv2_dolby', 'quality': 126, 'audio_quality': 30280,
+         'container': 'FLV', 'video_resolution': '3840p', 'desc': '杜比视界'},
         {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '3840p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,

From d6ae98fb1e0b2156e07f01957337fbd39a75f39d Mon Sep 17 00:00:00 2001
From: sdlyyxy <sdlyyxy@icloud.com>
Date: Sun, 6 Feb 2022 17:10:15 +0800
Subject: [PATCH 1075/1225] =?UTF-8?q?Bilibili:=20fix=20"=E7=9C=9F=E5=BD=A9?=
 =?UTF-8?q?=20HDR"=20video=5Fresolution?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5cd47e1050..101832e2dc 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -13,7 +13,7 @@ class Bilibili(VideoExtractor):
     # Bilibili media encoding options, in descending quality order.
     stream_types = [
         {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
-         'container': 'FLV', 'video_resolution': '3840p', 'desc': '真彩 HDR'},
+         'container': 'FLV', 'video_resolution': '2160p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '2160p', 'desc': '超清 4K'},
         {'id': 'flv_p60', 'quality': 116, 'audio_quality': 30280,

From 8d67fa7426d0fec0ac6144f5b3733ebfeb3fe574 Mon Sep 17 00:00:00 2001
From: bobo liu <liu@bobo.moe>
Date: Sat, 12 Feb 2022 20:21:15 +0800
Subject: [PATCH 1076/1225] change -safe to 0 for ffmpeg5

https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/46fb395952be32692385449ee214461603480b4b
---
 src/you_get/processor/ffmpeg.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 11126c2730..50e2c9fefb 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -93,7 +93,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
         concat_list = generate_concat_list(files, output)
-        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '0',
                                         '-i', concat_list, '-c', 'copy']
         params.extend(['--', output])
         if subprocess.call(params, stdin=STDIN) == 0:
@@ -149,7 +149,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
         concat_list = generate_concat_list(files, output)
-        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '0',
                                         '-i', concat_list, '-c', 'copy',
                                         '-bsf:a', 'aac_adtstoasc']
         params.extend(['--', output])
@@ -203,7 +203,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     # Use concat demuxer on FFmpeg >= 1.1
     if FFMPEG == 'ffmpeg' and (FFMPEG_VERSION[0] >= 2 or (FFMPEG_VERSION[0] == 1 and FFMPEG_VERSION[1] >= 1)):
         concat_list = generate_concat_list(files, output)
-        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '-1',
+        params = [FFMPEG] + LOGLEVEL + ['-y', '-f', 'concat', '-safe', '0',
                                         '-i', concat_list, '-c', 'copy',
                                         '-bsf:a', 'aac_adtstoasc']
         params.extend(['--', output])

From 3d5bd2b13d4988c3f55896cc6cdd8b34594642d2 Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Fri, 11 Mar 2022 02:38:44 +0800
Subject: [PATCH 1077/1225] update space_video_api arg: page size

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 5cd47e1050..49334d5bd7 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -121,7 +121,7 @@ def bilibili_space_favlist_api(fid, pn=1, ps=20):
         return 'https://api.bilibili.com/x/v3/fav/resource/list?media_id=%s&pn=%s&ps=%s&order=mtime&type=0&tid=0&jsonp=jsonp' % (fid, pn, ps)
 
     @staticmethod
-    def bilibili_space_video_api(mid, pn=1, ps=100):
+    def bilibili_space_video_api(mid, pn=1, ps=50):
         return "https://api.bilibili.com/x/space/arc/search?mid=%s&pn=%s&ps=%s&tid=0&keyword=&order=pubdate&jsonp=jsonp" % (mid, pn, ps)
 
     @staticmethod

From 699391135b517bbccbc295f9ecd6af77c2603955 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 18 Mar 2022 23:16:44 +0100
Subject: [PATCH 1078/1225] [instagram] fix

---
 src/you_get/extractors/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 0ed31cbc84..4167b22661 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -25,7 +25,7 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
         try:
             info = json.loads(data.group(1))
             post = info['entry_data']['PostPage'][0]
-            assert post
+            assert post['items']
         except:
             # with logged-in cookies
             data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', cont)

From 15393a8218faf96af88f506fa1a1a51a57fd31b3 Mon Sep 17 00:00:00 2001
From: liguangbin <nmslgb@chillcy.com>
Date: Sat, 19 Mar 2022 22:32:57 +0800
Subject: [PATCH 1079/1225] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=A5=BF=E7=93=9C?=
 =?UTF-8?q?=E8=A7=86=E9=A2=91=E4=B8=8B=E8=BD=BD=E5=A4=B1=E8=B4=A5=E9=97=AE?=
 =?UTF-8?q?=E9=A2=98;=20fix=20problem:the=20JSON=20object=20must=20be=20st?=
 =?UTF-8?q?r,=20bytes=20or=20bytearray,=20not=20NoneType?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/ixigua.py | 119 ++++++++++++++++++++-----------
 1 file changed, 77 insertions(+), 42 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 2f11e7f9d8..57119eb6d6 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -80,59 +80,94 @@ def tmp():
             return url
 
 
-def ixigua_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id='', **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
-    resp = urlopen_with_retry(request.Request(url))
+    headers['cookie'] = "MONITOR_WEB_ID=7892c49b-296e-4499-8704-e47c1b15123; " \
+                        "ixigua-a-s=1; ttcid=af99669b6304453480454f1507011d5c234; BD_REF=1; " \
+                        "__ac_nonce=060d88ff000a75e8d17eb; __ac_signature=_02B4Z6wo100f01kX9ZpgAAIDAKIBBQUIPYT5F2WIAAPG2ad; " \
+                        "ttwid=1%7CcIsVF_3vqSIk4XErhPB0H2VaTxT0tdsTMRbMjrJOPN8%7C1624806049%7C08ce7dd6f7d20506a41ba0a331ef96a6505d96731e6ad9f6c8c709f53f227ab1"
+
+    resp = urlopen_with_retry(request.Request(url, headers=headers))
     html = resp.read().decode('utf-8')
 
     _cookies = []
     for c in resp.getheader('Set-Cookie').split("httponly,"):
         _cookies.append(c.strip().split(' ')[0])
-    headers['cookie'] = ' '.join(_cookies)
+    headers['cookie'] += ';'.join(_cookies)
 
-    conf = loads(match1(html, r"window\.config = (.+);"))
-    if not conf:
-        log.e("Get window.config from url failed, url: {}".format(url))
+    match_txt = match1(html, r"<script id=\"SSR_HYDRATED_DATA\">window._SSR_HYDRATED_DATA=(.*?)<\/script>")
+    if not match_txt:
+        log.e("Get video info from url failed, url: {}".format(url))
         return
-    verify_url = conf['prefix'] + conf['url'] + '?key=' + conf['key'] + '&psm=' + conf['psm'] \
-        + '&_signature=' + ''.join(random.sample(string.ascii_letters + string.digits, 31))
-    try:
-        ok = get_content(verify_url)
-    except Exception as e:
-        ok = e.msg
-    if ok != 'OK':
-        log.e("Verify failed, verify_url: {}, result: {}".format(verify_url, ok))
+    video_info = loads(match_txt.replace('":undefined', '":null'))
+    if not video_info:
+        log.e("video_info not found, url:{}".format(url))
         return
-    html = get_content(url, headers=headers)
 
-    video_id = match1(html, r"\"vid\":\"([^\"]+)")
-    title = match1(html, r"\"player__videoTitle\">.*?<h1.*?>(.*)<\/h1><\/div>")
-    if not video_id:
-        log.e("video_id not found, url:{}".format(url))
-        return
-    video_info_url = get_video_url_from_video_id(video_id)
-    video_info = loads(get_content(video_info_url))
-    if video_info.get("code", 1) != 0:
-        log.e("Get video info from {} error: server return code {}".format(video_info_url, video_info.get("code", 1)))
-        return
-    if not video_info.get("data", None):
-        log.e("Get video info from {} error: The server returns JSON value"
-              " without data or data is empty".format(video_info_url))
+    title = video_info['anyVideo']['gidInformation']['packerData']['video']['title']
+    video_resource = video_info['anyVideo']['gidInformation']['packerData']['video']['videoResource']
+    if video_resource.get('dash', None):
+        video_list = video_resource['dash']
+    elif video_resource.get('dash_120fps', None):
+        video_list = video_resource['dash_120fps']
+    elif video_resource.get('normal', None):
+        video_list = video_resource['normal']
+    else:
+        log.e("video_list not found, url:{}".format(url))
         return
-    if not video_info["data"].get("video_list", None):
-        log.e("Get video info from {} error: The server returns JSON value"
-              " without data.video_list or data.video_list is empty".format(video_info_url))
-        return
-    if not video_info["data"]["video_list"].get("video_1", None):
-        log.e("Get video info from {} error: The server returns JSON value"
-              " without data.video_list.video_1 or data.video_list.video_1 is empty".format(video_info_url))
-        return
-    bestQualityVideo = list(video_info["data"]["video_list"].keys())[-1] #There is not only video_1, there might be video_2
-    size = int(video_info["data"]["video_list"][bestQualityVideo]["size"])
-    print_info(site_info=site_info, title=title, type="mp4", size=size)  # 该网站只有mp4类型文件
-    if not info_only:
-        video_url = base64.b64decode(video_info["data"]["video_list"][bestQualityVideo]["main_url"].encode("utf-8"))
-        download_urls([video_url.decode("utf-8")], title, "mp4", size, output_dir, merge=merge, headers=headers, **kwargs)
+
+    streams = [
+        # {'file_id': 'fc1b9bf8e8e04a849d90a5172d3f6919', 'quality': "normal", 'size': 0,
+        #  'definition': '720p', 'video_url': '','audio_url':'','v_type':'dash'},
+    ]
+    # 先用无水印的视频与音频合成，没有的话，再直接用有水印的mp4
+    if video_list.get('dynamic_video', None):
+        audio_url = base64.b64decode(
+            video_list['dynamic_video']['dynamic_audio_list'][0]['main_url'].encode("utf-8")).decode("utf-8")
+        dynamic_video_list = video_list['dynamic_video']['dynamic_video_list']
+        streams = convertStreams(dynamic_video_list, audio_url)
+    elif video_list.get('video_list', None):
+        dynamic_video_list = video_list['video_list']
+        streams = convertStreams(dynamic_video_list, "")
+
+    print("title:          %s" % title)
+    for stream in streams:
+        if stream_id != "" and stream_id != stream['definition']:
+            continue
+
+        print("    - format:        %s" % stream['definition'])
+        print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
+        print("      quality:          %s " % stream['quality'])
+        print("      v_type:          %s " % stream['v_type'])
+        # print("      video_url:          %s " % stream['video_url'])
+        # print("      audio_url:          %s " % stream['audio_url'])
+        print()
+
+        # 不是只看信息的话，就下载第一个
+        if not info_only:
+            urls = [stream['video_url']]
+            if stream['audio_url'] != "":
+                urls.append(stream['audio_url'])
+                kwargs['av'] = 'av'
+            download_urls(urls, title, "mp4", stream['size'], output_dir, merge=merge, headers=headers,
+                          **kwargs)
+            return
+
+
+def convertStreams(video_list, audio_url):
+    streams = []
+    for dynamic_video in video_list:
+        streams.append({
+            'file_id': dynamic_video['file_hash'],
+            'quality': dynamic_video['quality'],
+            'size': dynamic_video['size'],
+            'definition': dynamic_video['definition'],
+            'video_url': base64.b64decode(dynamic_video['main_url'].encode("utf-8")).decode("utf-8"),
+            'audio_url': audio_url,
+            'v_type': dynamic_video['vtype'],
+        })
+
+    return streams
 
 
 def ixigua_download_playlist_by_url(url, output_dir='.', merge=True, info_only=False, **kwargs):

From c8fa6c809731e8f144fbfd4337ab58986b25a8c2 Mon Sep 17 00:00:00 2001
From: liguangbin <nmslgb@chillcy.com>
Date: Sat, 19 Mar 2022 23:52:03 +0800
Subject: [PATCH 1080/1225] =?UTF-8?q?=E4=BC=98=E5=8C=96ixigua.py=E4=BB=A3?=
 =?UTF-8?q?=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/ixigua.py | 69 ++------------------------------
 1 file changed, 4 insertions(+), 65 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index 57119eb6d6..b4fdcefd9c 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -18,68 +18,6 @@
 }
 
 
-def int_overflow(val):
-    maxint = 2147483647
-    if not -maxint - 1 <= val <= maxint:
-        val = (val + (maxint + 1)) % (2 * (maxint + 1)) - maxint - 1
-    return val
-
-
-def unsigned_right_shitf(n, i):
-    if n < 0:
-        n = ctypes.c_uint32(n).value
-    if i < 0:
-        return -int_overflow(n << abs(i))
-    return int_overflow(n >> i)
-
-
-def get_video_url_from_video_id(video_id):
-    """Splicing URLs according to video ID to get video details"""
-    # from js
-    data = [""] * 256
-    for index, _ in enumerate(data):
-        t = index
-        for i in range(8):
-            t = -306674912 ^ unsigned_right_shitf(t, 1) if 1 & t else unsigned_right_shitf(t, 1)
-        data[index] = t
-
-    def tmp():
-        rand_num = random.random()
-        path = "/video/urls/v/1/toutiao/mp4/{video_id}?r={random_num}".format(video_id=video_id,
-                                                                              random_num=str(rand_num)[2:])
-        e = o = r = -1
-        i, a = 0, len(path)
-        while i < a:
-            e = ord(path[i])
-            i += 1
-            if e < 128:
-                r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ e)]
-            else:
-                if e < 2048:
-                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (192 | e >> 6 & 31))]
-                    r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]
-                else:
-                    if 55296 <= e < 57344:
-                        e = (1023 & e) + 64
-                        i += 1
-                        o = 1023 & t.url(i)
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (240 | e >> 8 & 7))]
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 2 & 63))]
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | o >> 6 & 15 | (3 & e) << 4))]
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & o))]
-                    else:
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (224 | e >> 12 & 15))]
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | e >> 6 & 63))]
-                        r = unsigned_right_shitf(r, 8) ^ data[255 & (r ^ (128 | 63 & e))]
-
-        return "https://ib.365yg.com{path}&s={param}".format(path=path, param=unsigned_right_shitf(r ^ -1, 0))
-
-    while 1:
-        url = tmp()
-        if url.split("=")[-1][0] != "-":  # 参数s不能为负数
-            return url
-
-
 def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id='', **kwargs):
     # example url: https://www.ixigua.com/i6631065141750268420/#mid=63024814422
     headers['cookie'] = "MONITOR_WEB_ID=7892c49b-296e-4499-8704-e47c1b15123; " \
@@ -137,8 +75,8 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id=
 
         print("    - format:        %s" % stream['definition'])
         print("      size:          %s MiB (%s bytes)" % (round(stream['size'] / 1048576, 1), stream['size']))
-        print("      quality:          %s " % stream['quality'])
-        print("      v_type:          %s " % stream['v_type'])
+        print("      quality:       %s " % stream['quality'])
+        print("      v_type:        %s " % stream['v_type'])
         # print("      video_url:          %s " % stream['video_url'])
         # print("      audio_url:          %s " % stream['audio_url'])
         print()
@@ -148,7 +86,8 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id=
             urls = [stream['video_url']]
             if stream['audio_url'] != "":
                 urls.append(stream['audio_url'])
-                kwargs['av'] = 'av'
+                kwargs['av'] = 'av'  # 这将会合并音视频
+
             download_urls(urls, title, "mp4", stream['size'], output_dir, merge=merge, headers=headers,
                           **kwargs)
             return

From f3fe2baa742bf3d09a8d4a6504bc33723e22339b Mon Sep 17 00:00:00 2001
From: liguangbin <nmslgb@chillcy.com>
Date: Sat, 19 Mar 2022 23:57:23 +0800
Subject: [PATCH 1081/1225] =?UTF-8?q?=E4=BC=98=E5=8C=96ixigua.py=E4=BB=A3?=
 =?UTF-8?q?=E7=A0=81?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 src/you_get/extractors/ixigua.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index b4fdcefd9c..b368b380ab 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -23,7 +23,7 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id=
     headers['cookie'] = "MONITOR_WEB_ID=7892c49b-296e-4499-8704-e47c1b15123; " \
                         "ixigua-a-s=1; ttcid=af99669b6304453480454f1507011d5c234; BD_REF=1; " \
                         "__ac_nonce=060d88ff000a75e8d17eb; __ac_signature=_02B4Z6wo100f01kX9ZpgAAIDAKIBBQUIPYT5F2WIAAPG2ad; " \
-                        "ttwid=1%7CcIsVF_3vqSIk4XErhPB0H2VaTxT0tdsTMRbMjrJOPN8%7C1624806049%7C08ce7dd6f7d20506a41ba0a331ef96a6505d96731e6ad9f6c8c709f53f227ab1"
+                        "ttwid=1%7CcIsVF_3vqSIk4XErhPB0H2VaTxT0tdsTMRbMjrJOPN8%7C1624806049%7C08ce7dd6f7d20506a41ba0a331ef96a6505d96731e6ad9f6c8c709f53f227ab1; "
 
     resp = urlopen_with_retry(request.Request(url, headers=headers))
     html = resp.read().decode('utf-8')
@@ -31,7 +31,7 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id=
     _cookies = []
     for c in resp.getheader('Set-Cookie').split("httponly,"):
         _cookies.append(c.strip().split(' ')[0])
-    headers['cookie'] += ';'.join(_cookies)
+    headers['cookie'] += ' '.join(_cookies)
 
     match_txt = match1(html, r"<script id=\"SSR_HYDRATED_DATA\">window._SSR_HYDRATED_DATA=(.*?)<\/script>")
     if not match_txt:

From a2ca1197892079e410f4056964d52ac4d723f05c Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Fri, 8 Apr 2022 02:48:36 +0800
Subject: [PATCH 1082/1225] clear stream info

---
 src/you_get/extractors/bilibili.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 49334d5bd7..c61a0567d0 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -141,6 +141,8 @@ def url_size(url, faker=False, headers={},err_value=0):
 
     def prepare(self, **kwargs):
         self.stream_qualities = {s['quality']: s for s in self.stream_types}
+        self.streams.clear()
+        self.dash_streams.clear()
 
         try:
             html_content = get_content(self.url, headers=self.bilibili_headers(referer=self.url))

From 5d41240860a33ac13b3ac3db7f4233904df19d9e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Apr 2022 23:20:38 +0200
Subject: [PATCH 1083/1225] [youtube] s_to_sig: no substitution for 3-char main
 function names

---
 src/you_get/extractors/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 81b45ac5b5..4e5e0f6068 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -78,6 +78,7 @@ def s_to_sig(js, s):
         # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
         # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
         # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
+        # - https://www.youtube.com/s/player/3b5d5649/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             # add prefix '_sig_' to prevent namespace pollution
@@ -117,7 +118,9 @@ def tr_js(code):
             f2 = re.sub(r'\$', '_dollar', f2)
             code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
 
-        f1 = re.sub(r'(as|if|in|is|or)', r'_\1', f1)
+        # if f1 contains more than 2 characters, no need to do substitution
+        # FIXME: we probably shouldn't do any substitution here at all?
+        f1 = re.sub(r'^(as|if|in|is|or)$', r'_\1', f1)
         f1 = re.sub(r'\$', '_dollar', f1)
         code = code + '_sig=_sig_%s(s)' % f1
         exec(code, globals(), locals())

From 16b75f0497b1f9c1ac6583e1335bf32b90211a8d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 8 Apr 2022 23:44:01 +0200
Subject: [PATCH 1084/1225] [youtube] support shorts

---
 src/you_get/extractors/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 4e5e0f6068..b5f56fa449 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -144,6 +144,7 @@ def get_vid_from_url(url):
         """
         return match1(url, r'youtu\.be/([^?/]+)') or \
           match1(url, r'youtube\.com/embed/([^/?]+)') or \
+          match1(url, r'youtube\.com/shorts/([^/?]+)') or \
           match1(url, r'youtube\.com/v/([^/?]+)') or \
           match1(url, r'youtube\.com/watch/([^/?]+)') or \
           parse_query_param(url, 'v') or \

From 9ef6c2ff6d88a00e4389f19126ce928da4136a03 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 15:34:26 +0200
Subject: [PATCH 1085/1225] [common] implement getHttps

---
 src/you_get/common.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 473c315592..afd6dad2d0 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -342,6 +342,23 @@ def undeflate(data):
     return decompressobj.decompress(data)+decompressobj.flush()
 
 
+# an http.client implementation of get_content()
+# because urllib does not support "Connection: keep-alive"
+def getHttps(host, url, headers, debuglevel=0):
+    import http.client
+
+    conn = http.client.HTTPSConnection(host)
+    conn.set_debuglevel(debuglevel)
+    conn.request("GET", url, headers=headers)
+    resp = conn.getresponse()
+
+    data = resp.read()
+    data = ungzip(data)
+    #data = undeflate(data)
+
+    return str(data, encoding='utf-8')
+
+
 # DEPRECATED in favor of get_content()
 def get_response(url, faker=False):
     logging.debug('get_response: %s' % url)

From ac2d02d1584d2de3ebbcdee19b7af7e6220abbbe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 15:41:39 +0200
Subject: [PATCH 1086/1225] [tests] fix test_bilibili

---
 tests/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index f3d45b97b2..2cdbefd66f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -41,7 +41,7 @@ def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
     def test_bilibili(self):
-        bilibili.download('https://space.bilibili.com/72270557/channel/seriesdetail?sid=218844', info_only=True)
+        bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 
     #def test_soundcloud(self):
         ## single song

From b34e66944e45671b59807a84ddde7e358e616c50 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 15:45:06 +0200
Subject: [PATCH 1087/1225] [tests] disable test_bilibili (works for me, dunno
 why)

---
 tests/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 2cdbefd66f..3099fadf91 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,8 +40,8 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    def test_bilibili(self):
-        bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
+    #def test_bilibili(self):
+    #    bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 
     #def test_soundcloud(self):
         ## single song

From db6ed38c6ad96caa23e98aaed5bad45f0fa437b3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 15:51:03 +0200
Subject: [PATCH 1088/1225] [tests] enable test_tiktok

---
 tests/test.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 3099fadf91..862b829f3d 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -53,10 +53,9 @@ def test_acfun(self):
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
         #)
 
-    #def tests_tiktok(self):
-    #    tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
-    #    tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
-    #    tiktok.download('https://vt.tiktok.com/UGJR4R/', info_only=True)
+    def test_tiktok(self):
+        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
 
 
 if __name__ == '__main__':

From 1b567d0830eb148faf964bcb797bb67f3581007b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 15:53:33 +0200
Subject: [PATCH 1089/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 56 ++++++++++++++------------------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 2ef0522666..d1069fccb4 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -5,42 +5,36 @@
 from ..common import *
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    referUrl = url.split('?')[0]
-    headers = fake_headers
-
-    # trick or treat
-    html = get_content(url, headers=headers)
-    data = r1(r'<script id="__NEXT_DATA__".*?>(.*?)</script>', html)
-    info = json.loads(data)
-    wid = info['props']['initialProps']['$wid']
-    cookie = 'tt_webid=%s; tt_webid_v2=%s' % (wid, wid)
-
-    # here's the cookie
-    headers['Cookie'] = cookie
-
-    # try again
-    html = get_content(url, headers=headers)
-    data = r1(r'<script id="__NEXT_DATA__".*?>(.*?)</script>', html)
+    while True:
+        m = re.match('https://([^/]+)(/.*)', url)
+        host = m.group(1)
+        if host == 'www.tiktok.com':  # canonical URL reached
+            url = m.group(2).split('?')[0]
+            vid = url.split('/')[3]  # should be a string of numbers
+            break
+        else:
+            url = get_location(url)
+
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
+        'Accept-Encoding': 'gzip, deflate',
+        'Accept': '*/*',
+        'Connection': 'keep-alive'  # important
+    }
+
+    html = getHttps(host, url, headers=headers)
+    data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html)
     info = json.loads(data)
-    wid = info['props']['initialProps']['$wid']
-    cookie = 'tt_webid=%s; tt_webid_v2=%s' % (wid, wid)
-
-    videoData = info['props']['pageProps']['itemInfo']['itemStruct']
-    videoId = videoData['id']
-    videoUrl = videoData['video']['downloadAddr']
-    uniqueId = videoData['author'].get('uniqueId')
-    nickName = videoData['author'].get('nickname')
-
-    title = '%s [%s]' % (nickName or uniqueId, videoId)
-
-    # we also need the referer
-    headers['Referer'] = referUrl
+    downloadAddr = info['ItemModule'][vid]['video']['downloadAddr']
+    author = info['ItemModule'][vid]['author']  # same as uniqueId
+    nickname = info['UserModule']['users'][author]['nickname']
+    title = '%s [%s]' % (nickname or author, vid)
 
-    mime, ext, size = url_info(videoUrl, headers=headers)
+    mime, ext, size = url_info(downloadAddr, headers=headers)
 
     print_info(site_info, title, mime, size)
     if not info_only:
-        download_urls([videoUrl], title, ext, size, output_dir=output_dir, merge=merge, headers=headers)
+        download_urls([downloadAddr], title, ext, size, output_dir=output_dir, merge=merge, headers=headers)
 
 site_info = "TikTok.com"
 download = tiktok_download

From 37ca277e7256139484195a699bd160540cddd8d8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 17:55:05 +0200
Subject: [PATCH 1090/1225] [bilibili] use hdflv2_hdr as id for HDR

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index da19eb6813..f7cc80dc6e 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -16,7 +16,7 @@ class Bilibili(VideoExtractor):
          'container': 'FLV', 'video_resolution': '4320p', 'desc': '超高清 8K'},
         {'id': 'hdflv2_dolby', 'quality': 126, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '3840p', 'desc': '杜比视界'},
-        {'id': 'hdflv2', 'quality': 125, 'audio_quality': 30280,
+        {'id': 'hdflv2_hdr', 'quality': 125, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '2160p', 'desc': '真彩 HDR'},
         {'id': 'hdflv2_4k', 'quality': 120, 'audio_quality': 30280,
          'container': 'FLV', 'video_resolution': '2160p', 'desc': '超清 4K'},

From 408e78b180557b2372249b370549685a50d4a787 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 19 Apr 2022 21:37:48 +0200
Subject: [PATCH 1091/1225] drop support for python <3.5

---
 README.md    | 2 +-
 you-get.json | 3 ---
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 0c3d4099ab..d4b8cd2946 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 The following dependencies are necessary:
 
-* **[Python](https://www.python.org/downloads/)**  3.2 or above
+* **[Python](https://www.python.org/downloads/)**  3.5 or above
 * **[FFmpeg](https://www.ffmpeg.org/)** 1.0 or above
 * (Optional) [RTMPDump](https://rtmpdump.mplayerhq.hu/)
 
diff --git a/you-get.json b/you-get.json
index a9ef1dd594..1a36b3c0d4 100644
--- a/you-get.json
+++ b/you-get.json
@@ -18,9 +18,6 @@
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.2",
-    "Programming Language :: Python :: 3.3",
-    "Programming Language :: Python :: 3.4",
     "Programming Language :: Python :: 3.5",
     "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",

From 43e14887a31704857452166c54c69c065f0b6036 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 19:00:29 +0200
Subject: [PATCH 1092/1225] [bilibili] support lowercase bv

---
 src/you_get/extractors/bilibili.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index f7cc80dc6e..48c919253f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -188,7 +188,7 @@ def prepare(self, **kwargs):
             sort = 'live'
         elif re.match(r'https?://vc\.bilibili\.com/video/(\d+)', self.url):
             sort = 'vc'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|(BV(\S+)))', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|(bv(\S+))|(BV(\S+)))', self.url):
             sort = 'video'
         elif re.match(r'https?://h\.?bilibili\.com/(\d+)', self.url):
             sort = 'h'
@@ -604,7 +604,7 @@ def download_playlist_by_url(self, url, **kwargs):
         elif re.match(r'https?://(www\.)?bilibili\.com/bangumi/media/md(\d+)', self.url) or \
             re.match(r'https?://bangumi\.bilibili\.com/anime/(\d+)', self.url):
             sort = 'bangumi_md'
-        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|BV(\S+))', self.url):
+        elif re.match(r'https?://(www\.)?bilibili\.com/video/(av(\d+)|bv(\S+)|BV(\S+))', self.url):
             sort = 'video'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/detail\?.*cid=(\d+)', self.url):
             sort = 'space_channel'

From 355e22584c8e8e0e5ac544945601f937355844cc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 19:17:26 +0200
Subject: [PATCH 1093/1225] [twitter] show the warning message if login
 required

---
 src/you_get/extractors/twitter.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 2346821147..8c052ed0db 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -51,7 +51,12 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
 
     info = json.loads(api_content)
-    if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+    if item_id not in info['globalObjects']['tweets']:
+        # something wrong here
+        log.w(info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'])
+        return
+
+    elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:
         # if the tweet contains media, download them
         media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
 

From bbc4df7a89ce32baa76e5d60d7a705354308b4f2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 19:29:52 +0200
Subject: [PATCH 1094/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index d1069fccb4..33e1f11e18 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -23,7 +23,8 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     }
 
     html = getHttps(host, url, headers=headers)
-    data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html)
+    data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html) or \
+        r1(r'<script id="SIGI_STATE" type="application/json">(.*?)</script>', html)
     info = json.loads(data)
     downloadAddr = info['ItemModule'][vid]['video']['downloadAddr']
     author = info['ItemModule'][vid]['author']  # same as uniqueId

From fd2d7fdcbc14384baf45c86588d769300e6bec79 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 23:21:14 +0200
Subject: [PATCH 1095/1225] [bilibili] support festival videos (fix #2955)

---
 src/you_get/extractors/bilibili.py | 52 ++++++++++++++++++++----------
 1 file changed, 35 insertions(+), 17 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 48c919253f..caaa91d663 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -177,6 +177,11 @@ def prepare(self, **kwargs):
             self.url = 'https://www.bilibili.com/%s' % match1(self.url, r'/s/(.+)')
             html_content = get_content(self.url, headers=self.bilibili_headers())
 
+        # redirect: festival
+        elif re.match(r'https?://(www\.)?bilibili\.com/festival/(.+)', self.url):
+            self.url = 'https://www.bilibili.com/video/%s' % match1(self.url, r'bvid=([^&]+)')
+            html_content = get_content(self.url, headers=self.bilibili_headers())
+
         # sort it out
         if re.match(r'https?://(www\.)?bilibili\.com/audio/au(\d+)', self.url):
             sort = 'audio'
@@ -203,30 +208,43 @@ def prepare(self, **kwargs):
 
             playinfo_text = match1(html_content, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo = json.loads(playinfo_text) if playinfo_text else None
-            playinfo = playinfo if playinfo['code'] == 0 else None
+            playinfo = playinfo if playinfo and playinfo.get('code') == 0 else None
 
             html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
             playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-            playinfo_ = playinfo_ if playinfo_['code'] == 0 else None
+            playinfo_ = playinfo_ if playinfo and playinfo_.get('code') == 0 else None
+
+            if 'videoData' in initial_state:
+                # (standard video)
+
+                # warn if it is a multi-part video
+                pn = initial_state['videoData']['videos']
+                if pn > 1 and not kwargs.get('playlist'):
+                    log.w('This is a multipart video. (use --playlist to download all parts.)')
+
+                # set video title
+                self.title = initial_state['videoData']['title']
+                # refine title for a specific part, if it is a multi-part video
+                p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
+                        '1')  # use URL to decide p-number, not initial_state['p']
+                if pn > 1:
+                    part = initial_state['videoData']['pages'][p - 1]['part']
+                    self.title = '%s (P%s. %s)' % (self.title, p, part)
+
+                # construct playinfos
+                avid = initial_state['aid']
+                cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
+            else:
+                # (festival video)
 
-            # warn if it is a multi-part video
-            pn = initial_state['videoData']['videos']
-            if pn > 1 and not kwargs.get('playlist'):
-                log.w('This is a multipart video. (use --playlist to download all parts.)')
+                # set video title
+                self.title = initial_state['videoInfo']['title']
 
-            # set video title
-            self.title = initial_state['videoData']['title']
-            # refine title for a specific part, if it is a multi-part video
-            p = int(match1(self.url, r'[\?&]p=(\d+)') or match1(self.url, r'/index_(\d+)') or
-                    '1')  # use URL to decide p-number, not initial_state['p']
-            if pn > 1:
-                part = initial_state['videoData']['pages'][p - 1]['part']
-                self.title = '%s (P%s. %s)' % (self.title, p, part)
+                # construct playinfos
+                avid = initial_state['videoInfo']['aid']
+                cid = initial_state['videoInfo']['cid']
 
-            # construct playinfos
-            avid = initial_state['aid']
-            cid = initial_state['videoData']['pages'][p - 1]['cid']  # use p-number, not initial_state['videoData']['cid']
             current_quality, best_quality = None, None
             if playinfo is not None:
                 current_quality = playinfo['data']['quality'] or None  # 0 indicates an error, fallback to None

From 0d5562497fd48245046a344560fbabbe0f5682d3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 23:24:50 +0200
Subject: [PATCH 1096/1225] [bilibili] oops

---
 src/you_get/extractors/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index caaa91d663..1a13b61cd2 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -213,7 +213,7 @@ def prepare(self, **kwargs):
             html_content_ = get_content(self.url, headers=self.bilibili_headers(cookie='CURRENT_FNVAL=16'))
             playinfo_text_ = match1(html_content_, r'__playinfo__=(.*?)</script><script>')  # FIXME
             playinfo_ = json.loads(playinfo_text_) if playinfo_text_ else None
-            playinfo_ = playinfo_ if playinfo and playinfo_.get('code') == 0 else None
+            playinfo_ = playinfo_ if playinfo_ and playinfo_.get('code') == 0 else None
 
             if 'videoData' in initial_state:
                 # (standard video)

From 62013d034f13d5483ffc05ac7d1d06acc8f77986 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 23:38:38 +0200
Subject: [PATCH 1097/1225] [common] fix google_search (close #2924)

---
 src/you_get/common.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 968d75441e..450a47d676 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1807,20 +1807,10 @@ def google_search(url):
     url = 'https://www.google.com/search?tbm=vid&q=%s' % parse.quote(keywords)
     page = get_content(url, headers=fake_headers)
     videos = re.findall(
-        r'<a href="(https?://[^"]+)" onmousedown="[^"]+"><h3 class="[^"]*">([^<]+)<', page
+        r'(https://www\.youtube\.com/watch\?v=[\w-]+)', page
     )
-    vdurs = re.findall(r'<span class="vdur[^"]*">([^<]+)<', page)
-    durs = [r1(r'(\d+:\d+)', unescape_html(dur)) for dur in vdurs]
-    print('Google Videos search:')
-    for v in zip(videos, durs):
-        print('- video:  {} [{}]'.format(
-            unescape_html(v[0][1]),
-            v[1] if v[1] else '?'
-        ))
-        print('# you-get %s' % log.sprint(v[0][0], log.UNDERLINE))
-        print()
     print('Best matched result:')
-    return(videos[0][0])
+    return(videos[0])
 
 
 def url_to_module(url):

From e2217bedd20bfb60db34c6aa82a5661a73c1603f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 22 Apr 2022 23:44:08 +0200
Subject: [PATCH 1098/1225] version 0.4.1602

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e8b65aae76..8fabc52e60 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1555'
+__version__ = '0.4.1602'

From 95d6d2e570bdd5b43bd55a758ac31ede2b5d298c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 29 Apr 2022 23:23:59 +0200
Subject: [PATCH 1099/1225] [tiktok] improve regexp

---
 src/you_get/extractors/tiktok.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 33e1f11e18..50780ac19e 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -6,10 +6,10 @@
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     while True:
-        m = re.match('https://([^/]+)(/.*)', url)
-        host = m.group(1)
+        m = re.match('(https?://)?([^/]+)(/.*)', url)
+        host = m.group(2)
         if host == 'www.tiktok.com':  # canonical URL reached
-            url = m.group(2).split('?')[0]
+            url = m.group(3).split('?')[0]
             vid = url.split('/')[3]  # should be a string of numbers
             break
         else:

From 0842913abfce8077b19394c07f9d68c775ee52b1 Mon Sep 17 00:00:00 2001
From: John Lu <johnlu1991@qq.com>
Date: Sun, 15 May 2022 22:20:15 +0800
Subject: [PATCH 1100/1225] Fixed a typo

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 450a47d676..82534a858f 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -1654,7 +1654,7 @@ def print_version():
     download_grp.add_argument('--itag', help=argparse.SUPPRESS)
 
     download_grp.add_argument('-m', '--m3u8', action='store_true', default=False,
-        help = 'download vide using an m3u8 url')
+        help = 'download video using an m3u8 url')
 
 
     parser.add_argument('URL', nargs='*', help=argparse.SUPPRESS)

From c768b291533129cdb0dcc8dcfbdfe3c6aa69d655 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 May 2022 22:50:09 +0200
Subject: [PATCH 1101/1225] [common] new parameters for getHttps

---
 src/you_get/common.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 82534a858f..c5c19d019d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -344,7 +344,7 @@ def undeflate(data):
 
 # an http.client implementation of get_content()
 # because urllib does not support "Connection: keep-alive"
-def getHttps(host, url, headers, debuglevel=0):
+def getHttps(host, url, headers, gzip=True, deflate=False, debuglevel=0):
     import http.client
 
     conn = http.client.HTTPSConnection(host)
@@ -353,8 +353,10 @@ def getHttps(host, url, headers, debuglevel=0):
     resp = conn.getresponse()
 
     data = resp.read()
-    data = ungzip(data)
-    #data = undeflate(data)
+    if gzip:
+        data = ungzip(data)
+    if deflate:
+        data = undeflate(data)
 
     return str(data, encoding='utf-8')
 

From e2ba3ecdb33e5fbc658611f903d9c778429b8cac Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 15 May 2022 22:51:22 +0200
Subject: [PATCH 1102/1225] [tiktok] fix extraction for alternative URLs

---
 src/you_get/extractors/tiktok.py | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 50780ac19e..09a36bb863 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -5,16 +5,6 @@
 from ..common import *
 
 def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    while True:
-        m = re.match('(https?://)?([^/]+)(/.*)', url)
-        host = m.group(2)
-        if host == 'www.tiktok.com':  # canonical URL reached
-            url = m.group(3).split('?')[0]
-            vid = url.split('/')[3]  # should be a string of numbers
-            break
-        else:
-            url = get_location(url)
-
     headers = {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
         'Accept-Encoding': 'gzip, deflate',
@@ -22,7 +12,20 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         'Connection': 'keep-alive'  # important
     }
 
+    m = re.match('(https?://)?([^/]+)(/.*)', url)
+    host = m.group(2)
+    if host != 'www.tiktok.com':  # non-canonical URL
+        html = getHttps(host, url, headers=headers, gzip=False)
+        url = r1(r'(https://www.tiktok.com/[^?"]+)', html)
+        # use canonical URL
+        m = re.match('(https?://)?([^/]+)(/.*)', url)
+        host = m.group(2)
+
+    url = m.group(3).split('?')[0]
+    vid = url.split('/')[3]  # should be a string of numbers
+
     html = getHttps(host, url, headers=headers)
+
     data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html) or \
         r1(r'<script id="SIGI_STATE" type="application/json">(.*?)</script>', html)
     info = json.loads(data)

From f2ea06473aabd2fcc598cd98aa7ceb93c95d978a Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 00:36:42 +0200
Subject: [PATCH 1103/1225] [tests] test_twitter

---
 tests/test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 862b829f3d..a1c6c07623 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -10,7 +10,8 @@
     acfun,
     bilibili,
     soundcloud,
-    tiktok
+    tiktok,
+    twitter
 )
 
 
@@ -28,7 +29,7 @@ def test_youtube(self):
         youtube.download(
             'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True
         )
-        youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True)
+        #youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True)
         #youtube.download(
         #    'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
         #    info_only=True
@@ -57,6 +58,9 @@ def test_tiktok(self):
         tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
         tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
 
+    def test_twitter(self):
+        twitter.download('https://twitter.com/elonmusk/status/1530516552084234244', info_only=True)
+
 
 if __name__ == '__main__':
     unittest.main()

From d57a0eba3e60cb4341a8bce02259ad8a4dee66c5 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 01:00:55 +0200
Subject: [PATCH 1104/1225] [youtube] improve regex

---
 src/you_get/extractors/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index b5f56fa449..3e1c5cadd1 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -237,7 +237,7 @@ def prepare(self, **kwargs):
 
                     except:
                         # ytplayer_config = {args:{raw_player_response:ytInitialPlayerResponse}}
-                        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+                        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
 
                         stream_list = ytInitialPlayerResponse['streamingData']['formats']
                         #stream_list = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
@@ -262,7 +262,7 @@ def prepare(self, **kwargs):
                 # Parse video page instead
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
 
-                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
 
                 self.title = ytInitialPlayerResponse["videoDetails"]["title"]
                 if re.search('([^"]*/base\.js)"', video_page):

From 249afb8b27498a89986d3af3aec2dad0819fa014 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 01:12:15 +0200
Subject: [PATCH 1105/1225] [.github/workflows] test pypy3.8 and pypy3.9

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 75231110ae..05dbc85af8 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3.8', 'pypy3.9']
 
     steps:
     - uses: actions/checkout@v2

From ce1f44fb88848c4e99357b51f6ab0b5cf9fe16a4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 01:14:47 +0200
Subject: [PATCH 1106/1225] [.github/workflows] test pypy-3.8 and pypy-3.9

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 05dbc85af8..47fb37a530 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3.8', 'pypy3.9']
+        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy-3.8, pypy-3.9]
 
     steps:
     - uses: actions/checkout@v2

From 1aa7ca21fcc6769859c9b072f1a5052d73f546d9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 01:41:25 +0200
Subject: [PATCH 1107/1225] drop support for python <3.7.4

---
 .github/workflows/python-package.yml | 2 +-
 README.md                            | 8 +++++---
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 47fb37a530..f90b61aea8 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9, '3.10', pypy-3.8, pypy-3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', pypy-3.8, pypy-3.9]
 
     steps:
     - uses: actions/checkout@v2
diff --git a/README.md b/README.md
index d4b8cd2946..e0cea4dd51 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,9 @@
 [![PyPI version](https://img.shields.io/pypi/v/you-get.svg)](https://pypi.python.org/pypi/you-get/)
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-**NOTICE: Read [this](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) if you are looking for the conventional "Issues" tab.**
+**NOTICE (30 May 2022): Support for Python 3.5, 3.6 and 3.7 will eventually be dropped. ([see details here](https://github.com/soimort/you-get/wiki/TLS-1.3-post-handshake-authentication-(PHA)))**
+
+**NOTICE (8 Mar 2019): Read [this](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) if you are looking for the conventional "Issues" tab.**
 
 ---
 
@@ -53,9 +55,9 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 ### Prerequisites
 
-The following dependencies are necessary:
+The following dependencies are recommended:
 
-* **[Python](https://www.python.org/downloads/)**  3.5 or above
+* **[Python](https://www.python.org/downloads/)**  3.8 or above
 * **[FFmpeg](https://www.ffmpeg.org/)** 1.0 or above
 * (Optional) [RTMPDump](https://rtmpdump.mplayerhq.hu/)
 

From a5c726b9d701f81f4cb976242baeb5f00a7c164b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 01:44:36 +0200
Subject: [PATCH 1108/1225] version 0.4.1612

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 8fabc52e60..da7d3c336b 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1602'
+__version__ = '0.4.1612'

From 9980b727cbcad548059cd05fbfa0254cc99b8e26 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 14:50:20 +0200
Subject: [PATCH 1109/1225] update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index e0cea4dd51..44c102cde8 100644
--- a/README.md
+++ b/README.md
@@ -57,7 +57,7 @@ Are you a Python programmer? Then check out [the source](https://github.com/soim
 
 The following dependencies are recommended:
 
-* **[Python](https://www.python.org/downloads/)**  3.8 or above
+* **[Python](https://www.python.org/downloads/)**  3.7.4 or above
 * **[FFmpeg](https://www.ffmpeg.org/)** 1.0 or above
 * (Optional) [RTMPDump](https://rtmpdump.mplayerhq.hu/)
 

From 6268c1173ce183b2548850bf95d1e7587ad22019 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 14:53:21 +0200
Subject: [PATCH 1110/1225] update README.rst

---
 README.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.rst b/README.rst
index 3c23ab5e50..376abcf70e 100644
--- a/README.rst
+++ b/README.rst
@@ -52,7 +52,7 @@ source <https://github.com/soimort/you-get>`__ and fork it!
 
 .. |PyPI version| image:: https://badge.fury.io/py/you-get.png
    :target: http://badge.fury.io/py/you-get
-.. |Build Status| image:: https://api.travis-ci.org/soimort/you-get.png
-   :target: https://travis-ci.org/soimort/you-get
+.. |Build Status| image:: https://github.com/soimort/you-get/workflows/develop/badge.svg
+   :target: https://github.com/soimort/you-get/actions
 .. |Gitter| image:: https://badges.gitter.im/Join%20Chat.svg
    :target: https://gitter.im/soimort/you-get?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge

From bfb35db5a6b6dfd31c55c2c2f3edd56c0c59e8bf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 15:38:00 +0200
Subject: [PATCH 1111/1225] update you-get.json

---
 you-get.json | 2 --
 1 file changed, 2 deletions(-)

diff --git a/you-get.json b/you-get.json
index 1a36b3c0d4..bb94ba0041 100644
--- a/you-get.json
+++ b/you-get.json
@@ -18,8 +18,6 @@
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.5",
-    "Programming Language :: Python :: 3.6",
     "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",

From 6ddc3fce89bb496394ab6f51c224b0f9964ee344 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 30 May 2022 15:38:47 +0200
Subject: [PATCH 1112/1225] update Makefile

---
 Makefile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c0f9cf0e25..fe4a238c14 100644
--- a/Makefile
+++ b/Makefile
@@ -43,5 +43,7 @@ install:
 	$(SETUP) install --user --prefix=
 
 release:
-	zenity --question
-	$(SETUP) sdist bdist_wheel upload --sign
+	#zenity --question
+	$(SETUP) sdist bdist_wheel
+	echo 'Upload new version to PyPI using:'
+	echo '	twine upload --sign dist/you-get-VERSION.tar.gz dist/you_get-VERSION-py3-none-any.whl'

From c0151a97756990bca525598dc37db476cd6c34d4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 7 Jun 2022 16:19:14 +0200
Subject: [PATCH 1113/1225] [youtube] we should extract ytInitialPlayerResponse
 more reliably

---
 src/you_get/extractors/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 3e1c5cadd1..f820152fa1 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -237,7 +237,10 @@ def prepare(self, **kwargs):
 
                     except:
                         # ytplayer_config = {args:{raw_player_response:ytInitialPlayerResponse}}
-                        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
+                        try:  # FIXME: we should extract ytInitialPlayerResponse more reliably
+                            ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
+                        except:
+                            ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
 
                         stream_list = ytInitialPlayerResponse['streamingData']['formats']
                         #stream_list = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
@@ -262,7 +265,10 @@ def prepare(self, **kwargs):
                 # Parse video page instead
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
 
-                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
+                try:  # FIXME: we should extract ytInitialPlayerResponse more reliably
+                    ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
+                except:
+                    ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
 
                 self.title = ytInitialPlayerResponse["videoDetails"]["title"]
                 if re.search('([^"]*/base\.js)"', video_page):

From a47960f6ed7b2a484b6629678b3a6ad8e39497bd Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 20 Jun 2022 23:04:56 +0200
Subject: [PATCH 1114/1225] [twitter] better warning

---
 src/you_get/extractors/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 8c052ed0db..19b4ce879b 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -53,7 +53,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     info = json.loads(api_content)
     if item_id not in info['globalObjects']['tweets']:
         # something wrong here
-        log.w(info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'])
+        log.wtf('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'], exit_code=None)
         return
 
     elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:

From d661c95480abd61f7ef8877d8dbcb827534aa54d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2022 22:21:47 +0200
Subject: [PATCH 1115/1225] [instagram] fix extraction

---
 src/you_get/extractors/instagram.py | 78 +++++++++++++----------------
 1 file changed, 34 insertions(+), 44 deletions(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 4167b22661..604c534cec 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -10,60 +10,50 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
     description = r1(r'<meta property="og:title" content="([^"]*)"', cont) or \
-        r1(r'<title>\s([^<]*)</title>', cont) # with logged-in cookies
+        r1(r'<title>([^<]*)</title>', cont) # with logged-in cookies
     title = "{} [{}]".format(description.replace("\n", " "), vid)
 
-    stream = r1(r'<meta property="og:video" content="([^"]*)"', cont)
-    if stream:
-        _, ext, size = url_info(stream)
-
-        print_info(site_info, title, ext, size)
-        if not info_only:
-            download_urls([stream], title, ext, size, output_dir, merge=merge)
-    else:
-        data = re.search(r'window\._sharedData\s*=\s*(.*);</script>', cont)
-        try:
-            info = json.loads(data.group(1))
-            post = info['entry_data']['PostPage'][0]
-            assert post['items']
-        except:
-            # with logged-in cookies
-            data = re.search(r'window\.__additionalDataLoaded\(\'[^\']+\',(.*)\);</script>', cont)
-            if data is not None:
-                log.e('[Warning] Cookies needed.')
-            post = json.loads(data.group(1))
-
-        for item in post['items']:
-            code = item['code']
-            carousel_media = item.get('carousel_media') or [item]
-            for i, media in enumerate(carousel_media):
-                title = '%s [%s]' % (code, i)
-                image_url = media['image_versions2']['candidates'][0]['url']
-                ext = image_url.split('?')[0].split('.')[-1]
-                size = int(get_head(image_url)['Content-Length'])
+    appId = r1(r'"appId":"(\d+)"', cont)
+    media_id = r1(r'"media_id":"(\d+)"', cont)
+
+    api_url = 'https://i.instagram.com/api/v1/media/%s/info/' % media_id
+    try:
+        api_cont = get_content(api_url, headers={**fake_headers, **{'x-ig-app-id': appId}})
+    except:
+        log.wtf('[Error] Please specify a cookie file.')
+    post = json.loads(api_cont)
+
+    for item in post['items']:
+        code = item['code']
+        carousel_media = item.get('carousel_media') or [item]
+        for i, media in enumerate(carousel_media):
+            title = '%s [%s]' % (code, i)
+            image_url = media['image_versions2']['candidates'][0]['url']
+            ext = image_url.split('?')[0].split('.')[-1]
+            size = int(get_head(image_url)['Content-Length'])
+
+            print_info(site_info, title, ext, size)
+            if not info_only:
+                download_urls(urls=[image_url],
+                              title=title,
+                              ext=ext,
+                              total_size=size,
+                              output_dir=output_dir)
+
+            # download videos (if any)
+            if 'video_versions' in media:
+                video_url = media['video_versions'][0]['url']
+                ext = video_url.split('?')[0].split('.')[-1]
+                size = int(get_head(video_url)['Content-Length'])
 
                 print_info(site_info, title, ext, size)
                 if not info_only:
-                    download_urls(urls=[image_url],
+                    download_urls(urls=[video_url],
                                   title=title,
                                   ext=ext,
                                   total_size=size,
                                   output_dir=output_dir)
 
-                # download videos (if any)
-                if 'video_versions' in media:
-                    video_url = media['video_versions'][0]['url']
-                    ext = video_url.split('?')[0].split('.')[-1]
-                    size = int(get_head(video_url)['Content-Length'])
-
-                    print_info(site_info, title, ext, size)
-                    if not info_only:
-                        download_urls(urls=[video_url],
-                                      title=title,
-                                      ext=ext,
-                                      total_size=size,
-                                      output_dir=output_dir)
-
 site_info = "Instagram.com"
 download = instagram_download
 download_playlist = playlist_not_supported('instagram')

From 82b376a0c60ff473686d6a79ae6ca5c42dc93950 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 1 Jul 2022 23:26:50 +0200
Subject: [PATCH 1116/1225] version 0.4.1620

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index da7d3c336b..ac2bfc0342 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1612'
+__version__ = '0.4.1620'

From 4119a1493e3c1c46c04914accd677d331c357edb Mon Sep 17 00:00:00 2001
From: owlwang <owlwang@outlook.com>
Date: Mon, 25 Jul 2022 12:34:55 +0800
Subject: [PATCH 1117/1225] fix douyin extractor

---
 src/you_get/extractors/douyin.py | 51 ++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 8067b1b564..6a59b16074 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -1,8 +1,6 @@
 # coding=utf-8
 
-import re
 import json
-from urllib.parse import unquote
 
 from ..common import (
     url_size,
@@ -11,25 +9,52 @@
     fake_headers,
     download_urls,
     playlist_not_supported,
+    match1,
+    get_location,
 )
 
-
 __all__ = ['douyin_download_by_url']
 
 
+def get_value(source: dict, path):
+    try:
+        value = source
+        for key in path:
+            if type(key) is str:
+                if key in value.keys():
+                    value = value[key]
+                else:
+                    value = None
+                    break
+            elif type(key) is int:
+                if len(value) != 0:
+                    value = value[key]
+                else:
+                    value = None
+                    break
+    except:
+        value = None
+    return value
+
+
 def douyin_download_by_url(url, **kwargs):
+    # if short link, get the real url
+    if 'v.douyin.com' in url:
+        url = get_location(url)
+    aweme_id = match1(url, r'/(\d+)/?')
+    # get video info
+    video_info_api = 'https://www.douyin.com/web/api/v2/aweme/iteminfo/?item_ids={}'
+    url = video_info_api.format(aweme_id)
     page_content = get_content(url, headers=fake_headers)
-    # The video player and video source are rendered client-side, the data
-    # contains in a <script id="RENDER_DATA" type="application/json"> tag
-    # quoted, unquote the whole page content then search using regex with
-    # regular string.
-    page_content = unquote(page_content)
-    title = re.findall(r'"desc":"([^"]*)"', page_content)[0].strip()
+    video_info = json.loads(page_content)
+
+    # get video id and title
+    video_id = get_value(video_info, ['item_list', 0, 'video', 'vid'])
+    title = get_value(video_info, ['item_list', 0, 'desc'])
+
+    # get video play url
+    video_url = "https://aweme.snssdk.com/aweme/v1/playwm/?ratio=720p&line=0&video_id={}".format(video_id)
     video_format = 'mp4'
-    # video URLs are in this pattern {"src":"THE_URL"}, in json format
-    urls_pattern = r'"playAddr":(\[.*?\])'
-    urls = json.loads(re.findall(urls_pattern, page_content)[0])
-    video_url = 'https:' + urls[0]['src']
     size = url_size(video_url, faker=True)
     print_info(
         site_info='douyin.com', title=title,

From 5e25e1b5e29e66884e1ff20f965deee4898eb07e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Aug 2022 15:20:34 +0200
Subject: [PATCH 1118/1225] [youtube] fix s_to_sig

---
 src/you_get/extractors/youtube.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index f820152fa1..ddf12be953 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -79,6 +79,7 @@ def s_to_sig(js, s):
         # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
         # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
         # - https://www.youtube.com/s/player/3b5d5649/player_ias.vflset/sv_SE/base.js
+        # - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
         def tr_js(code):
             code = re.sub(r'function', r'def', code)
             # add prefix '_sig_' to prevent namespace pollution
@@ -114,14 +115,10 @@ def tr_js(code):
             else:
                 f2def = re.search(r'[^$\w]%s:function\((\w+)\)(\{[^\{\}]+\})' % f2e, js)
                 f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
-            f2 = re.sub(r'(as|if|in|is|or)', r'_\1', f2)
-            f2 = re.sub(r'\$', '_dollar', f2)
+            f2 = re.sub(r'\$', '_dollar', f2)  # replace dollar sign
             code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
 
-        # if f1 contains more than 2 characters, no need to do substitution
-        # FIXME: we probably shouldn't do any substitution here at all?
-        f1 = re.sub(r'^(as|if|in|is|or)$', r'_\1', f1)
-        f1 = re.sub(r'\$', '_dollar', f1)
+        f1 = re.sub(r'\$', '_dollar', f1)  # replace dollar sign
         code = code + '_sig=_sig_%s(s)' % f1
         exec(code, globals(), locals())
         return locals()['_sig']

From fd5e31309792127d44d0601c493af1f6ac684efc Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 25 Aug 2022 15:36:11 +0200
Subject: [PATCH 1119/1225] [tiktok] fix extraction for alternative URLs

---
 src/you_get/extractors/tiktok.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 09a36bb863..b5a6d4bf24 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -15,9 +15,7 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     m = re.match('(https?://)?([^/]+)(/.*)', url)
     host = m.group(2)
     if host != 'www.tiktok.com':  # non-canonical URL
-        html = getHttps(host, url, headers=headers, gzip=False)
-        url = r1(r'(https://www.tiktok.com/[^?"]+)', html)
-        # use canonical URL
+        url = get_location(url, headers=headers)
         m = re.match('(https?://)?([^/]+)(/.*)', url)
         host = m.group(2)
 

From 9f7892b73f9d1fac9f47c6350153f8bc314fae2a Mon Sep 17 00:00:00 2001
From: Qing <navylq@163.com>
Date: Fri, 9 Sep 2022 16:25:51 +0800
Subject: [PATCH 1120/1225] Subtitute the ffmpeg download method.

Subtitute the ffmpeg download method.
---
 src/you_get/extractors/iqiyi.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index d138a49fb8..b8fb84190d 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -203,8 +203,13 @@ def download(self, **kwargs):
             # For legacy main()
 
             #Here's the change!!
-            download_url_ffmpeg(urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], stream=False)
-
+            # ffmpeg fails to parse.
+            # download_url_ffmpeg(urls[0], self.title, 'mp4', output_dir=kwargs['output_dir'], merge=kwargs['merge'], stream=False)
+            #Here's the way works out
+            urls = general_m3u8_extractor(urls[0])
+            # ffmpeg fail to convert the output video with mkv extension, due to sort of timestamp problem
+            download_urls(urls, self.title, 'mp4', 0, **kwargs)
+            
             if not kwargs['caption']:
                 print('Skipping captions.')
                 return

From 9a6878ee6eb63377e8906e490a457cd1bcca5fbc Mon Sep 17 00:00:00 2001
From: Qing <navylq@163.com>
Date: Mon, 12 Sep 2022 10:22:37 +0800
Subject: [PATCH 1121/1225] narrow the search of tvid and videoid

test script `python3 you-get https://www.iqiyi.com/a_19rrha613h.html -l --debug`
---
 src/you_get/extractors/iqiyi.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index b8fb84190d..16bf45d3a3 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -131,10 +131,10 @@ def prepare(self, **kwargs):
             html = get_html(self.url)
             tvid = r1(r'#curid=(.+)_', self.url) or \
                    r1(r'tvid=([^&]+)', self.url) or \
-                   r1(r'data-player-tvid="([^"]+)"', html) or r1(r'tv(?:i|I)d=(.+?)\&', html) or r1(r'param\[\'tvid\'\]\s*=\s*"(.+?)"', html)
+                   r1(r'data-player-tvid="([^"]+)"', html) or r1(r'tv(?:i|I)d=(\w+?)\&', html) or r1(r'param\[\'tvid\'\]\s*=\s*"(.+?)"', html)
             videoid = r1(r'#curid=.+_(.*)$', self.url) or \
                       r1(r'vid=([^&]+)', self.url) or \
-                      r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(.+?)\&', html) or r1(r'param\[\'vid\'\]\s*=\s*"(.+?)"', html)
+                      r1(r'data-player-videoid="([^"]+)"', html) or r1(r'vid=(\w+?)\&', html) or r1(r'param\[\'vid\'\]\s*=\s*"(.+?)"', html)
             self.vid = (tvid, videoid)
             info_u = 'http://pcw-api.iqiyi.com/video/video/playervideoinfo?tvid=' + tvid
             json_res = get_content(info_u)

From af3468fa059e6f9ae9450066a70b110127312c75 Mon Sep 17 00:00:00 2001
From: interiv <interiv@qq.com>
Date: Tue, 13 Sep 2022 17:27:12 +0800
Subject: [PATCH 1122/1225] Update youku.py

ccode 0532 to 0564
---
 src/you_get/extractors/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 92cdafb6ec..ed0743bbc0 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -77,7 +77,7 @@ def __init__(self):
         self.api_error_code = None
         self.api_error_msg = None
 
-        self.ccode = '0532'
+        self.ccode = '0564'
         # Found in http://g.alicdn.com/player/ykplayer/0.5.64/youku-player.min.js
         # grep -oE '"[0-9a-zA-Z+/=]{256}"' youku-player.min.js
         self.ckey = 'DIl58SLFxFNndSV1GFNnMQVYkx1PP5tKe1siZu/86PR1u/Wh1Ptd+WOZsHHWxysSfAOhNJpdVWsdVJNsfJ8Sxd8WKVvNfAS8aS8fAOzYARzPyPc3JvtnPHjTdKfESTdnuTW6ZPvk2pNDh4uFzotgdMEFkzQ5wZVXl2Pf1/Y6hLK0OnCNxBj3+nb0v72gZ6b0td+WOZsHHWxysSo/0y9D2K42SaB8Y/+aD2K42SaB8Y/+ahU+WOZsHcrxysooUeND'

From e4358725b1364f9ec94d3574cf9a99c4e9ff4a28 Mon Sep 17 00:00:00 2001
From: Christian Clauss <cclauss@me.com>
Date: Sat, 17 Sep 2022 18:30:25 +0200
Subject: [PATCH 1123/1225] Add Python 3.11 release candidate 2 to the testing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Python 3.11 will be on average 22% faster than Python 3.10 so let’s give it a spin...
https://www.python.org/download/pre-releases
---
 .github/workflows/python-package.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index f90b61aea8..37a8f1aad3 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -16,12 +16,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', pypy-3.8, pypy-3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', 3.11-dev, pypy-3.8, pypy-3.9]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

From b81b4ae8ddd8edcbd1a4c12f6480e5f4857abf79 Mon Sep 17 00:00:00 2001
From: xiangxw <xiangxw5689@126.com>
Date: Tue, 20 Sep 2022 11:11:18 +0800
Subject: [PATCH 1124/1225] [ixigua] fix 'string indices must be integers'

---
 src/you_get/extractors/ixigua.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index b368b380ab..f2fd953e69 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -95,6 +95,8 @@ def ixigua_download(url, output_dir='.', merge=True, info_only=False, stream_id=
 
 def convertStreams(video_list, audio_url):
     streams = []
+    if type(video_list) == dict:
+        video_list = video_list.values()
     for dynamic_video in video_list:
         streams.append({
             'file_id': dynamic_video['file_hash'],

From 023d12fa2d049d1de53605cca3678b6ce45886ea Mon Sep 17 00:00:00 2001
From: David Dai <stdioa@163.com>
Date: Thu, 22 Sep 2022 20:34:28 +0800
Subject: [PATCH 1125/1225] [netease]: Add pagination on netease radio

---
 src/you_get/extractors/netease.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/netease.py b/src/you_get/extractors/netease.py
index ca1be887b8..48e6dc532c 100644
--- a/src/you_get/extractors/netease.py
+++ b/src/you_get/extractors/netease.py
@@ -79,9 +79,14 @@ def netease_cloud_music_download(url, output_dir='.', merge=True, info_only=Fals
         netease_song_download(j["program"]["mainSong"], output_dir=output_dir, info_only=info_only)
 
     elif "radio" in url:
-        j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))
-        for i in j['programs']:
-            netease_song_download(i["mainSong"],output_dir=output_dir, info_only=info_only)
+        offset = 0
+        while True:
+            j = loads(get_content("http://music.163.com/api/dj/program/byradio/?radioId=%s&ids=[%s]&csrf_token=&offset=%d" % (rid, rid, offset), headers={"Referer": "http://music.163.com/"}))
+            for i in j['programs']:
+                netease_song_download(i["mainSong"], output_dir=output_dir, info_only=info_only)
+            if not j['more']:
+                break
+            offset += len(j['programs'])
 
     elif "mv" in url:
         j = loads(get_content("http://music.163.com/api/mv/detail/?id=%s&ids=[%s]&csrf_token=" % (rid, rid), headers={"Referer": "http://music.163.com/"}))

From 11dcced98bd842b41896d6e8f7edc711434ea12c Mon Sep 17 00:00:00 2001
From: 0x4ea5 <0x4ea5@gmail.com>
Date: Sun, 25 Sep 2022 19:01:51 -0700
Subject: [PATCH 1126/1225] added support to bilibili collectiondetail api

---
 src/you_get/extractors/bilibili.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 1a13b61cd2..6d34c2c45f 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -115,11 +115,15 @@ def bilibili_live_room_init_api(room_id):
     @staticmethod
     def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
+   
+    @staticmethod
+    def bilibili_space_collection_api(mid, cid, pn=1, ps=30):
+        return 'https://api.bilibili.com/x/polymer/space/seasons_archives_list?mid=%s&season_id=%s&sort_reverse=false&page_num=%s&page_size=%s' % (mid, cid, pn, ps)
 
     @staticmethod
     def bilibili_series_archives_api(mid, sid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/series/archives?mid=%s&series_id=%s&pn=%s&ps=%s&only_normal=true&sort=asc&jsonp=jsonp' % (mid, sid, pn, ps)
-
+    
     @staticmethod
     def bilibili_space_favlist_api(fid, pn=1, ps=20):
         return 'https://api.bilibili.com/x/v3/fav/resource/list?media_id=%s&pn=%s&ps=%s&order=mtime&type=0&tid=0&jsonp=jsonp' % (fid, pn, ps)
@@ -628,6 +632,8 @@ def download_playlist_by_url(self, url, **kwargs):
             sort = 'space_channel'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/seriesdetail\?.*sid=(\d+)', self.url):
             sort = 'space_channel_series'
+        elif re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/collectiondetail\?.*sid=(\d+)', self.url):
+            sort = 'space_channel_collection'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url):
             sort = 'space_favlist'
         elif re.match(r'https?://space\.?bilibili\.com/(\d+)/video', self.url):
@@ -752,6 +758,20 @@ def download_playlist_by_url(self, url, **kwargs):
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
 
+        elif sort == 'space_channel_collection':
+            m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/collectiondetail\?.*sid=(\d+)', self.url)
+            mid, sid = m.group(1), m.group(2)
+            api_url = self.bilibili_space_collection_api(mid, sid)
+            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+            archives_info = json.loads(api_content)
+            # TBD: channel of more than 100 videos
+
+            epn, i = len(archives_info['data']['archives']), 0
+            for video in archives_info['data']['archives']:
+                i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
+                url = 'https://www.bilibili.com/video/av%s' % video['aid']
+                self.__class__().download_playlist_by_url(url, **kwargs)
+
         elif sort == 'space_favlist':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/favlist\?.*fid=(\d+)', self.url)
             vmid, fid = m.group(1), m.group(2)

From f4e36b366749df87e0ceccac050862b495424f51 Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Wed, 5 Oct 2022 01:23:32 +0800
Subject: [PATCH 1127/1225] fix: use fake ua

---
 src/you_get/extractors/missevan.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index b7f413f20a..31fbbd62a4 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -310,7 +310,7 @@ def extract(self, **kwargs):
                 or kwargs.get('json_output'):
 
             for _, stream in self.streams.items():
-                stream['size'] = urls_size(stream['src'])
+                stream['size'] = urls_size(stream['src'], faker=True)
             return
 
         # fetch size of the selected stream only
@@ -319,7 +319,7 @@ def extract(self, **kwargs):
 
         stream = self.streams[stream_id]
         if 'size' not in stream:
-            stream['size'] = urls_size(stream['src'])
+            stream['size'] = urls_size(stream['src'], faker=True)
 
     def _get_content(self, url):
         return get_content(url, headers=self.__headers)

From 568ee3e8d25d110127912f6220d4300d45021b67 Mon Sep 17 00:00:00 2001
From: nuomi1 <nuomi1@qq.com>
Date: Wed, 5 Oct 2022 01:24:53 +0800
Subject: [PATCH 1128/1225] fix: remove query and fragment

---
 src/you_get/extractors/missevan.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/missevan.py b/src/you_get/extractors/missevan.py
index 31fbbd62a4..6d9533cef7 100644
--- a/src/you_get/extractors/missevan.py
+++ b/src/you_get/extractors/missevan.py
@@ -25,6 +25,7 @@
 import json
 import os
 import re
+import urllib.parse
 
 from ..common import get_content, urls_size, log, player, dry_run
 from ..extractor import VideoExtractor
@@ -99,7 +100,8 @@ def is_covers_stream(stream):
     return stream.lower() in ('covers', 'coversmini')
 
 def get_file_extension(file_path, default=''):
-    _, suffix = os.path.splitext(file_path)
+    url_parse_result = urllib.parse.urlparse(file_path)
+    _, suffix = os.path.splitext(url_parse_result.path)
     if suffix:
         # remove dot
         suffix = suffix[1:]

From 81b1f06a05564195204b5dbb83a20005528b2b7d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 9 Oct 2022 00:26:20 +0200
Subject: [PATCH 1129/1225] [twitter] support tweets with multiple videos

---
 src/you_get/extractors/twitter.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 19b4ce879b..4e4adeac39 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -82,16 +82,16 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
 
     for medium in media:
         if 'video_info' in medium:
-            # FIXME: we're assuming one tweet only contains one video here
             variants = medium['video_info']['variants']
             variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
+            title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0]
             urls = [ variants[-1]['url'] ]
             size = urls_size(urls)
             mime, ext = variants[-1]['content_type'], 'mp4'
 
-            print_info(site_info, page_title, mime, size)
+            print_info(site_info, title, mime, size)
             if not info_only:
-                download_urls(urls, page_title, ext, size, output_dir, merge=merge)
+                download_urls(urls, title, ext, size, output_dir, merge=merge)
 
         else:
             title = item_id + '_' + medium['media_url_https'].split('.')[-2].split('/')[-1]

From 4746af346218e446220111d304db434fe3dcd4cf Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 9 Oct 2022 00:37:48 +0200
Subject: [PATCH 1130/1225] [twitter] small fix

---
 src/you_get/extractors/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 4e4adeac39..efa6f28718 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -84,7 +84,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         if 'video_info' in medium:
             variants = medium['video_info']['variants']
             variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
-            title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0]
+            title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0].split('.')[0]
             urls = [ variants[-1]['url'] ]
             size = urls_size(urls)
             mime, ext = variants[-1]['content_type'], 'mp4'

From d7d3237bf94415becac7d88c4a4645623b51fc41 Mon Sep 17 00:00:00 2001
From: zheng zhiwen <zhiwen.zh@gmail.com>
Date: Mon, 17 Oct 2022 10:05:34 +0800
Subject: [PATCH 1131/1225] test miaopai

---
 tests/test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/test.py b/tests/test.py
index a1c6c07623..877b693531 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -11,7 +11,8 @@
     bilibili,
     soundcloud,
     tiktok,
-    twitter
+    twitter,
+    miaopai
 )
 
 
@@ -61,6 +62,8 @@ def test_tiktok(self):
     def test_twitter(self):
         twitter.download('https://twitter.com/elonmusk/status/1530516552084234244', info_only=True)
 
+    def test_weibo(self):
+        miaopai.download('https://video.weibo.com/show?fid=1034:4825403706245135', info_only=True)
 
 if __name__ == '__main__':
     unittest.main()

From dc7dc481f45d15f178e5982dc405d8b6a61f06f5 Mon Sep 17 00:00:00 2001
From: Andy Fan <eaao@rlsnk.com>
Date: Mon, 17 Oct 2022 14:15:50 +0800
Subject: [PATCH 1132/1225] [douyin] remove video watermark

---
 src/you_get/extractors/douyin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/douyin.py b/src/you_get/extractors/douyin.py
index 6a59b16074..4b60de3c55 100644
--- a/src/you_get/extractors/douyin.py
+++ b/src/you_get/extractors/douyin.py
@@ -53,7 +53,7 @@ def douyin_download_by_url(url, **kwargs):
     title = get_value(video_info, ['item_list', 0, 'desc'])
 
     # get video play url
-    video_url = "https://aweme.snssdk.com/aweme/v1/playwm/?ratio=720p&line=0&video_id={}".format(video_id)
+    video_url = "https://aweme.snssdk.com/aweme/v1/play/?ratio=720p&line=0&video_id={}".format(video_id)
     video_format = 'mp4'
     size = url_size(video_url, faker=True)
     print_info(

From 5b22081a9cfa46beba8ece3e7f91e0d6d5cde95d Mon Sep 17 00:00:00 2001
From: zheng zhiwen <zhiwen.zh@gmail.com>
Date: Mon, 17 Oct 2022 16:10:30 +0800
Subject: [PATCH 1133/1225] miaopai fix

---
 src/you_get/extractors/miaopai.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 05c1e650b3..91a301b832 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -80,6 +80,8 @@ def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **
 
 def miaopai_download_h5api(url, output_dir='.', merge=False, info_only=False, **kwargs):
     oid = match1(url, r'/show/(\d{4}:\w+)')
+    if oid is None:
+        oid = match1(url, r'\?fid=(\d{4}:\w+)')
     page = "/show/%s" % oid
     data_url = 'https://h5.video.weibo.com/api/component?%s' % parse.urlencode({
         'page': page
@@ -156,6 +158,9 @@ def miaopai_download(url, output_dir='.', merge=False, info_only=False, **kwargs
     if re.match(r'^http[s]://(.+\.)?weibo\.com/(tv/)?show/(\d{4}:\w+)', url):
         return miaopai_download_h5api(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 
+    if re.match(r'^http[s]://(.+\.)?weibo\.com/show\?fid=(\d{4}:\w+)', url):
+        return miaopai_download_h5api(url, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
+
     fid = match1(url, r'\?fid=(\d{4}:\w+)')
     if fid is not None:
         miaopai_download_by_fid(fid, output_dir, merge, info_only)

From 9f608990ee21717d55a6941b9bad739e5e0ce923 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Oct 2022 17:10:51 +0200
Subject: [PATCH 1134/1225] [twitter] support NSFW tweets

---
 src/you_get/extractors/twitter.py | 69 +++++++++++++++++++------------
 1 file changed, 42 insertions(+), 27 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index efa6f28718..24a8c12cb3 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -41,44 +41,59 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         r1(r'<meta name="twitter:site:id" content="([^"]*)"', html)
     page_title = "{} [{}]".format(screen_name, item_id)
 
-    authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
+    try:
+        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
 
-    ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
-    ga_content = post_content(ga_url, headers={'authorization': authorization})
-    guest_token = json.loads(ga_content)['guest_token']
-
-    api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
-    api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
-
-    info = json.loads(api_content)
-    if item_id not in info['globalObjects']['tweets']:
-        # something wrong here
-        log.wtf('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'], exit_code=None)
-        return
-
-    elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:
-        # if the tweet contains media, download them
-        media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
-
-    elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
-        # if the tweet does not contain media, but it quotes a tweet
-        # and the quoted tweet contains media, download them
-        item_id = info['globalObjects']['tweets'][item_id]['quoted_status_id_str']
+        # FIXME: 403 with cookies
+        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
+        ga_content = post_content(ga_url, headers={'authorization': authorization})
+        guest_token = json.loads(ga_content)['guest_token']
 
         api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
         api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
 
         info = json.loads(api_content)
+        if item_id not in info['globalObjects']['tweets']:
+            # something wrong here
+            #log.wtf('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'], exit_code=None)
+            assert False
 
-        if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+        elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+            # if the tweet contains media, download them
             media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+
+        elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
+            # if the tweet does not contain media, but it quotes a tweet
+            # and the quoted tweet contains media, download them
+            item_id = info['globalObjects']['tweets'][item_id]['quoted_status_id_str']
+
+            api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
+            api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
+
+            info = json.loads(api_content)
+
+            if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
+                media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
+            else:
+                # quoted tweet has no media
+                return
+
         else:
-            # quoted tweet has no media
+            # no media, no quoted tweet
             return
 
-    else:
-        # no media, no quoted tweet
-        return
+    except:
+        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw'
+
+        # FIXME: 403 with cookies
+        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
+        ga_content = post_content(ga_url, headers={'authorization': authorization})
+        guest_token = json.loads(ga_content)['guest_token']
+
+        api_url = 'https://api.twitter.com/1.1/statuses/show/%s.json?tweet_mode=extended' % item_id
+        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
+        info = json.loads(api_content)
+        media = info['extended_entities']['media']
 
     for medium in media:
         if 'video_info' in medium:

From 8ef66a31736adf2b0b8928fa94eb8e800f25e7c8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Oct 2022 18:46:33 +0200
Subject: [PATCH 1135/1225] [twitter] if the tweet contains video from another
 tweet, download it

---
 src/you_get/extractors/twitter.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 24a8c12cb3..c80eb21187 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -62,6 +62,13 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             # if the tweet contains media, download them
             media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
 
+        elif 'entities' in info['globalObjects']['tweets'][item_id]:
+            # if the tweet contains video from another tweet, download it
+            # FIXME: multiple urls?
+            expanded_url = info['globalObjects']['tweets'][item_id]['entities']['urls'][0]['expanded_url']
+            item_id = r1(r'/(\d+)/video', expanded_url)
+            assert False
+
         elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
             # if the tweet does not contain media, but it quotes a tweet
             # and the quoted tweet contains media, download them

From e674bfbc2b1faf2c14efde0d5aba79979e9db06f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 25 Oct 2022 13:49:15 +0200
Subject: [PATCH 1136/1225] [twitter] small fix

---
 src/you_get/extractors/twitter.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index c80eb21187..7975bdfde9 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -63,11 +63,15 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
 
         elif 'entities' in info['globalObjects']['tweets'][item_id]:
-            # if the tweet contains video from another tweet, download it
-            # FIXME: multiple urls?
-            expanded_url = info['globalObjects']['tweets'][item_id]['entities']['urls'][0]['expanded_url']
-            item_id = r1(r'/(\d+)/video', expanded_url)
-            assert False
+            # if the tweet contains media from another tweet, download it
+            expanded_url = None
+            for j in info['globalObjects']['tweets'][item_id]['entities']['urls']:
+                if re.match(r'^https://twitter.com/.*', j['expanded_url']):
+                    # FIXME: multiple valid expanded_url's?
+                    expanded_url = j['expanded_url']
+            if expanded_url is not None:
+                item_id = r1(r'/status/(\d+)', expanded_url)
+                assert False
 
         elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
             # if the tweet does not contain media, but it quotes a tweet

From 84db11759e05e4b6ee525806743a2f1b1aae4f90 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Dec 2022 17:08:34 +0100
Subject: [PATCH 1137/1225] [tiktok] fix extraction

---
 src/you_get/common.py            | 2 +-
 src/you_get/extractors/tiktok.py | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c5c19d019d..1558baf69d 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -358,7 +358,7 @@ def getHttps(host, url, headers, gzip=True, deflate=False, debuglevel=0):
     if deflate:
         data = undeflate(data)
 
-    return str(data, encoding='utf-8')
+    return str(data, encoding='utf-8'), resp.getheader('set-cookie')
 
 
 # DEPRECATED in favor of get_content()
diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index b5a6d4bf24..641e5e9744 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -9,6 +9,7 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
         'Accept-Encoding': 'gzip, deflate',
         'Accept': '*/*',
+        'Referer': 'https://www.tiktok.com/',
         'Connection': 'keep-alive'  # important
     }
 
@@ -22,7 +23,9 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = m.group(3).split('?')[0]
     vid = url.split('/')[3]  # should be a string of numbers
 
-    html = getHttps(host, url, headers=headers)
+    html, set_cookie = getHttps(host, url, headers=headers)
+    tt_chain_token = r1('tt_chain_token=([^;]+);', set_cookie)
+    headers['Cookie'] = 'tt_chain_token=%s' % tt_chain_token
 
     data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html) or \
         r1(r'<script id="SIGI_STATE" type="application/json">(.*?)</script>', html)

From 7b845b34ce18863e519ad3cce8e53431ba41664d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Dec 2022 17:43:07 +0100
Subject: [PATCH 1138/1225] [tiktok] fix extraction for alternative URLs

---
 src/you_get/common.py            | 15 +++++++++------
 src/you_get/extractors/tiktok.py | 12 ++++++------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 1558baf69d..c337a2a2a3 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -344,21 +344,24 @@ def undeflate(data):
 
 # an http.client implementation of get_content()
 # because urllib does not support "Connection: keep-alive"
-def getHttps(host, url, headers, gzip=True, deflate=False, debuglevel=0):
+def getHttps(host, url, headers, debuglevel=0):
     import http.client
 
     conn = http.client.HTTPSConnection(host)
     conn.set_debuglevel(debuglevel)
     conn.request("GET", url, headers=headers)
     resp = conn.getresponse()
+    set_cookie = resp.getheader('set-cookie')
 
     data = resp.read()
-    if gzip:
-        data = ungzip(data)
-    if deflate:
-        data = undeflate(data)
+    try:
+        data = ungzip(data)  # gzip
+        data = undeflate(data)  # deflate
+    except:
+        pass
 
-    return str(data, encoding='utf-8'), resp.getheader('set-cookie')
+    conn.close()
+    return str(data, encoding='utf-8'), set_cookie
 
 
 # DEPRECATED in favor of get_content()
diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 641e5e9744..2c4892f629 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -16,12 +16,12 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     m = re.match('(https?://)?([^/]+)(/.*)', url)
     host = m.group(2)
     if host != 'www.tiktok.com':  # non-canonical URL
-        url = get_location(url, headers=headers)
-        m = re.match('(https?://)?([^/]+)(/.*)', url)
-        host = m.group(2)
-
-    url = m.group(3).split('?')[0]
-    vid = url.split('/')[3]  # should be a string of numbers
+        vid = r1(r'/video/(\d+)', url)
+        url = 'https://www.tiktok.com/@/video/%s/' % vid
+        host = 'www.tiktok.com'
+    else:
+        url = m.group(3).split('?')[0]
+        vid = url.split('/')[3]  # should be a string of numbers
 
     html, set_cookie = getHttps(host, url, headers=headers)
     tt_chain_token = r1('tt_chain_token=([^;]+);', set_cookie)

From 888a9e29f37a52a57a81b28bebdf39bd77aa058f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Dec 2022 17:44:06 +0100
Subject: [PATCH 1139/1225] [tests] test "universal" tiktok url

---
 tests/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test.py b/tests/test.py
index 877b693531..c0f3836aed 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -57,6 +57,7 @@ def test_acfun(self):
 
     def test_tiktok(self):
         tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+        tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
         tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
 
     def test_twitter(self):

From fd7889783419940da9ed460ab420c48be39a2ae4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Dec 2022 18:09:10 +0100
Subject: [PATCH 1140/1225] [instagram] show cookie warning

---
 src/you_get/extractors/instagram.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 604c534cec..8e261fe791 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -19,9 +19,9 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
     api_url = 'https://i.instagram.com/api/v1/media/%s/info/' % media_id
     try:
         api_cont = get_content(api_url, headers={**fake_headers, **{'x-ig-app-id': appId}})
+        post = json.loads(api_cont)
     except:
         log.wtf('[Error] Please specify a cookie file.')
-    post = json.loads(api_cont)
 
     for item in post['items']:
         code = item['code']

From 0d9c28031010ba44fc69977050d5fe572fdee12b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 11 Dec 2022 18:15:16 +0100
Subject: [PATCH 1141/1225] version 0.4.1650

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index ac2bfc0342..440488a9ac 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1620'
+__version__ = '0.4.1650'

From c0a483dab1a07bce353a8cb8f6cb4111c6348a85 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 18 Dec 2022 14:54:34 +0100
Subject: [PATCH 1142/1225] [twitter] warn when falling back to deprecated API

---
 src/you_get/extractors/twitter.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 7975bdfde9..baf4c37557 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -55,7 +55,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
         info = json.loads(api_content)
         if item_id not in info['globalObjects']['tweets']:
             # something wrong here
-            #log.wtf('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'], exit_code=None)
+            log.w('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'])
             assert False
 
         elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:
@@ -94,6 +94,8 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             return
 
     except:
+        log.w('[Warning] Falling back to deprecated Twitter API. Extraction may be incomplete.')
+
         authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw'
 
         # FIXME: 403 with cookies

From 0fc63efa63c88662f363fa89768b1c1f9dd6cc36 Mon Sep 17 00:00:00 2001
From: arix00 <15333224+arix00@users.noreply.github.com>
Date: Sun, 1 Jan 2023 20:38:21 -0800
Subject: [PATCH 1143/1225] Download multipage video collection

When there're more than single page videos in a collection,
Download all videos as current code will only handle first page.

For 'space_channel_series' and 'space_channel_collection'
---
 src/you_get/extractors/bilibili.py | 38 ++++++++++++++++++++----------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 6d34c2c45f..b082553e9b 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -747,13 +747,20 @@ def download_playlist_by_url(self, url, **kwargs):
         elif sort == 'space_channel_series':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/seriesdetail\?.*sid=(\d+)', self.url)
             mid, sid = m.group(1), m.group(2)
-            api_url = self.bilibili_series_archives_api(mid, sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            archives_info = json.loads(api_content)
-            # TBD: channel of more than 100 videos
+            pn = 1
+            video_list = []
+            while True:
+                api_url = self.bilibili_series_archives_api(mid, sid, pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                archives_info = json.loads(api_content)
+                video_list.extend(archives_info['data']['archives'])
+                if len(video_list) < archives_info['data']['page']['total'] and len(archives_info['data']['archives']) > 0:
+                    pn += 1
+                else:
+                    break
 
-            epn, i = len(archives_info['data']['archives']), 0
-            for video in archives_info['data']['archives']:
+            epn, i = len(video_list), 0
+            for video in video_list:
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)
@@ -761,13 +768,20 @@ def download_playlist_by_url(self, url, **kwargs):
         elif sort == 'space_channel_collection':
             m = re.match(r'https?://space\.?bilibili\.com/(\d+)/channel/collectiondetail\?.*sid=(\d+)', self.url)
             mid, sid = m.group(1), m.group(2)
-            api_url = self.bilibili_space_collection_api(mid, sid)
-            api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
-            archives_info = json.loads(api_content)
-            # TBD: channel of more than 100 videos
+            pn = 1
+            video_list = []
+            while True:
+                api_url = self.bilibili_space_collection_api(mid, sid, pn)
+                api_content = get_content(api_url, headers=self.bilibili_headers(referer=self.url))
+                archives_info = json.loads(api_content)
+                video_list.extend(archives_info['data']['archives'])
+                if len(video_list) < archives_info['data']['page']['total'] and len(archives_info['data']['archives']) > 0:
+                    pn += 1
+                else:
+                    break
 
-            epn, i = len(archives_info['data']['archives']), 0
-            for video in archives_info['data']['archives']:
+            epn, i = len(video_list), 0
+            for video in video_list:
                 i += 1; log.w('Extracting %s of %s videos ...' % (i, epn))
                 url = 'https://www.bilibili.com/video/av%s' % video['aid']
                 self.__class__().download_playlist_by_url(url, **kwargs)

From 25eb89984524acd42a9b704d3d5b0edfa509c95a Mon Sep 17 00:00:00 2001
From: juruoyyx <60863833+juruoyyx@users.noreply.github.com>
Date: Fri, 6 Jan 2023 10:55:00 +0800
Subject: [PATCH 1144/1225] Update ffmpeg.py

---
 src/you_get/processor/ffmpeg.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 50e2c9fefb..efc0a47280 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -128,7 +128,7 @@ def ffmpeg_concat_mp4_to_mpg(files, output='output.mpg'):
 
 def ffmpeg_concat_ts_to_mkv(files, output='output.mkv'):
     print('Merging video parts... ', end="", flush=True)
-    params = [FFMPEG] + LOGLEVEL + ['-isync', '-y', '-i']
+    params = [FFMPEG] + LOGLEVEL + ['-y', '-i']
     params.append('concat:')
     for file in files:
         if os.path.isfile(file):

From a2e411395b9bffa0329c3ea4d80c8fbb218e7bad Mon Sep 17 00:00:00 2001
From: URenko <18209292+URenko@users.noreply.github.com>
Date: Wed, 8 Feb 2023 18:51:01 +0800
Subject: [PATCH 1145/1225] support different codecs for bilibili

---
 src/you_get/extractors/bilibili.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 6d34c2c45f..6ec8bc1333 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -42,6 +42,8 @@ class Bilibili(VideoExtractor):
         {'id': 'jpg', 'quality': 0},
     ]
 
+    codecids = {7: 'AVC', 12: 'HEVC', 13: 'AV1'}
+
     @staticmethod
     def height_to_quality(height, qn):
         if height <= 360 and qn <= 16:
@@ -70,7 +72,7 @@ def bilibili_headers(referer=None, cookie=None):
 
     @staticmethod
     def bilibili_api(avid, cid, qn=0):
-        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=16&fourk=1' % (avid, cid, qn)
+        return 'https://api.bilibili.com/x/player/playurl?avid=%s&cid=%s&qn=%s&type=&otype=json&fnver=0&fnval=4048&fourk=1' % (avid, cid, qn)
 
     @staticmethod
     def bilibili_audio_api(sid):
@@ -302,11 +304,10 @@ def prepare(self, **kwargs):
                 if 'dash' in playinfo['data']:
                     audio_size_cache = {}
                     for video in playinfo['data']['dash']['video']:
-                        # prefer the latter codecs!
                         s = self.stream_qualities[video['id']]
-                        format_id = 'dash-' + s['id']  # prefix
+                        format_id = f"dash-{s['id']}-{self.codecids[video['codecid']]}"  # prefix
                         container = 'mp4'  # enforce MP4 container
-                        desc = s['desc']
+                        desc = s['desc'] + ' ' + video['codecs']
                         audio_quality = s['audio_quality']
                         baseurl = video['baseUrl']
                         size = self.url_size(baseurl, headers=self.bilibili_headers(referer=self.url))

From f54669411e5b10b2e79484f0d07f00664b450bc0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 8 Feb 2023 16:41:03 +0100
Subject: [PATCH 1146/1225] add param "--prefix" to prefix downloaded files

---
 src/you_get/common.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index c337a2a2a3..bdb67bac77 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -138,6 +138,7 @@
 insecure = False
 m3u8 = False
 postfix = False
+prefix = None
 
 fake_headers = {
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
@@ -1014,6 +1015,8 @@ def download_urls(
     title = tr(get_filename(title))
     if postfix and 'vid' in kwargs:
         title = "%s [%s]" % (title, kwargs['vid'])
+    if prefix is not None:
+        title = "[%s] %s" % (prefix, title)
     output_filename = get_output_filename(urls, title, ext, output_dir, merge)
     output_filepath = os.path.join(output_dir, output_filename)
 
@@ -1563,9 +1566,13 @@ def print_version():
         help='Do not download captions (subtitles, lyrics, danmaku, ...)'
     )
     download_grp.add_argument(
-        '--postfix', action='store_true', default=False,
+        '--post', '--postfix', dest='postfix', action='store_true', default=False,
         help='Postfix downloaded files with unique identifiers'
     )
+    download_grp.add_argument(
+        '--pre', '--prefix', dest='prefix', metavar='PREFIX', default=None,
+        help='Prefix downloaded files with string'
+    )
     download_grp.add_argument(
         '-f', '--force', action='store_true', default=False,
         help='Force overwriting existing files'
@@ -1689,6 +1696,7 @@ def print_version():
     global insecure
     global m3u8
     global postfix
+    global prefix
     output_filename = args.output_filename
     extractor_proxy = args.extractor_proxy
 
@@ -1726,6 +1734,7 @@ def print_version():
         insecure = True
 
     postfix = args.postfix
+    prefix = args.prefix
 
     if args.no_proxy:
         set_http_proxy('')

From 2ba7493f126aed5785893b4cd5c3042998da7b99 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 9 Feb 2023 23:36:16 +0100
Subject: [PATCH 1147/1225] [bilibili] warn if cookies are not loaded

---
 src/you_get/extractors/bilibili.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index e59296ee61..6335e6dd87 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -117,7 +117,7 @@ def bilibili_live_room_init_api(room_id):
     @staticmethod
     def bilibili_space_channel_api(mid, cid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/space/channel/video?mid=%s&cid=%s&pn=%s&ps=%s&order=0&jsonp=jsonp' % (mid, cid, pn, ps)
-   
+
     @staticmethod
     def bilibili_space_collection_api(mid, cid, pn=1, ps=30):
         return 'https://api.bilibili.com/x/polymer/space/seasons_archives_list?mid=%s&season_id=%s&sort_reverse=false&page_num=%s&page_size=%s' % (mid, cid, pn, ps)
@@ -125,7 +125,7 @@ def bilibili_space_collection_api(mid, cid, pn=1, ps=30):
     @staticmethod
     def bilibili_series_archives_api(mid, sid, pn=1, ps=100):
         return 'https://api.bilibili.com/x/series/archives?mid=%s&series_id=%s&pn=%s&ps=%s&only_normal=true&sort=asc&jsonp=jsonp' % (mid, sid, pn, ps)
-    
+
     @staticmethod
     def bilibili_space_favlist_api(fid, pn=1, ps=20):
         return 'https://api.bilibili.com/x/v3/fav/resource/list?media_id=%s&pn=%s&ps=%s&order=mtime&type=0&tid=0&jsonp=jsonp' % (fid, pn, ps)
@@ -224,6 +224,10 @@ def prepare(self, **kwargs):
             if 'videoData' in initial_state:
                 # (standard video)
 
+                # warn if cookies are not loaded
+                if cookies is None:
+                    log.w('You will need login cookies for 720p formats or above. (use --cookies to load cookies.txt.)')
+
                 # warn if it is a multi-part video
                 pn = initial_state['videoData']['videos']
                 if pn > 1 and not kwargs.get('playlist'):

From 2aaa877a9b8ebda9ed25cb87df1ad760700b55c4 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 15 Feb 2023 16:20:47 +0100
Subject: [PATCH 1148/1225] [.github/workflows] test python 3.11

---
 .github/workflows/python-package.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 37a8f1aad3..39793c03f7 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -1,5 +1,4 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: develop
 
@@ -16,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', 3.11-dev, pypy-3.8, pypy-3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', pypy-3.8, pypy-3.9]
 
     steps:
     - uses: actions/checkout@v3

From ad5825a8f644442a3f45e028b7f04f4c6d861aba Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 9 May 2023 15:22:19 +0200
Subject: [PATCH 1149/1225] [twitter] fix extraction

---
 src/you_get/extractors/twitter.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index baf4c37557..752ef74608 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -34,7 +34,18 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    html = get_html(url, faker=True) # now it seems faker must be enabled
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
+    }
+    host = 'www.twitter.com'
+
+    html, set_cookie = getHttps(host, url, headers=headers)
+    # "Found. Redirecting to..."
+    guest_id = r1('guest_id=([^;]+);', set_cookie)
+    headers['Cookie'] = 'guest_id=%s' % guest_id
+
+    html = get_content(url, headers=headers)
+
     screen_name = r1(r'twitter\.com/([^/]+)', url) or r1(r'data-screen-name="([^"]*)"', html) or \
         r1(r'<meta name="twitter:title" content="([^"]*)"', html)
     item_id = r1(r'twitter\.com/[^/]+/status/(\d+)', url) or r1(r'data-item-id="([^"]*)"', html) or \

From e733351a059ae85a1fd4d101655bca8c8bf4279c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 5 Jul 2023 17:12:15 +0200
Subject: [PATCH 1150/1225] [twitter] fix extraction

---
 src/you_get/extractors/twitter.py | 123 +++++++-----------------------
 1 file changed, 28 insertions(+), 95 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 752ef74608..43cfa6a4b1 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -23,7 +23,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     if re.match(r'https?://mobile', url): # normalize mobile URL
         url = 'https://' + match1(url, r'//mobile\.(.+)')
 
-    if re.match(r'https?://twitter\.com/i/moments/', url): # moments
+    if re.match(r'https?://twitter\.com/i/moments/', url): # FIXME: moments
         html = get_html(url, faker=True)
         paths = re.findall(r'data-permalink-path="([^"]+)"', html)
         for path in paths:
@@ -34,114 +34,47 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0'
-    }
-    host = 'www.twitter.com'
-
-    html, set_cookie = getHttps(host, url, headers=headers)
-    # "Found. Redirecting to..."
-    guest_id = r1('guest_id=([^;]+);', set_cookie)
-    headers['Cookie'] = 'guest_id=%s' % guest_id
+    m = re.match('^https?://(mobile\.)?twitter\.com/([^/]+)/status/(\d+)', url)
+    assert m
+    screen_name, item_id = m.group(2), m.group(3)
+    page_title = "{} [{}]".format(screen_name, item_id)
 
-    html = get_content(url, headers=headers)
+    # FIXME: this API won't work for protected or nsfw contents
+    api_url = 'https://cdn.syndication.twimg.com/tweet-result?id=%s' % item_id
+    content = get_content(api_url)
+    info = json.loads(content)
 
-    screen_name = r1(r'twitter\.com/([^/]+)', url) or r1(r'data-screen-name="([^"]*)"', html) or \
-        r1(r'<meta name="twitter:title" content="([^"]*)"', html)
-    item_id = r1(r'twitter\.com/[^/]+/status/(\d+)', url) or r1(r'data-item-id="([^"]*)"', html) or \
-        r1(r'<meta name="twitter:site:id" content="([^"]*)"', html)
-    page_title = "{} [{}]".format(screen_name, item_id)
+    author = info['user']['name']
+    url = 'https://twitter.com/%s/status/%s' % (info['user']['screen_name'], item_id)
+    full_text = info['text']
 
-    try:
-        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
-
-        # FIXME: 403 with cookies
-        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
-        ga_content = post_content(ga_url, headers={'authorization': authorization})
-        guest_token = json.loads(ga_content)['guest_token']
-
-        api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
-        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
-
-        info = json.loads(api_content)
-        if item_id not in info['globalObjects']['tweets']:
-            # something wrong here
-            log.w('[Failed] ' + info['timeline']['instructions'][0]['addEntries']['entries'][0]['content']['item']['content']['tombstone']['tombstoneInfo']['richText']['text'])
-            assert False
-
-        elif 'extended_entities' in info['globalObjects']['tweets'][item_id]:
-            # if the tweet contains media, download them
-            media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
-
-        elif 'entities' in info['globalObjects']['tweets'][item_id]:
-            # if the tweet contains media from another tweet, download it
-            expanded_url = None
-            for j in info['globalObjects']['tweets'][item_id]['entities']['urls']:
-                if re.match(r'^https://twitter.com/.*', j['expanded_url']):
-                    # FIXME: multiple valid expanded_url's?
-                    expanded_url = j['expanded_url']
-            if expanded_url is not None:
-                item_id = r1(r'/status/(\d+)', expanded_url)
-                assert False
-
-        elif info['globalObjects']['tweets'][item_id].get('is_quote_status') == True:
-            # if the tweet does not contain media, but it quotes a tweet
-            # and the quoted tweet contains media, download them
-            item_id = info['globalObjects']['tweets'][item_id]['quoted_status_id_str']
-
-            api_url = 'https://api.twitter.com/2/timeline/conversation/%s.json?tweet_mode=extended' % item_id
-            api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
-
-            info = json.loads(api_content)
-
-            if 'extended_entities' in info['globalObjects']['tweets'][item_id]:
-                media = info['globalObjects']['tweets'][item_id]['extended_entities']['media']
-            else:
-                # quoted tweet has no media
-                return
-
-        else:
-            # no media, no quoted tweet
-            return
-
-    except:
-        log.w('[Warning] Falling back to deprecated Twitter API. Extraction may be incomplete.')
-
-        authorization = 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw'
-
-        # FIXME: 403 with cookies
-        ga_url = 'https://api.twitter.com/1.1/guest/activate.json'
-        ga_content = post_content(ga_url, headers={'authorization': authorization})
-        guest_token = json.loads(ga_content)['guest_token']
-
-        api_url = 'https://api.twitter.com/1.1/statuses/show/%s.json?tweet_mode=extended' % item_id
-        api_content = get_content(api_url, headers={'authorization': authorization, 'x-guest-token': guest_token})
-        info = json.loads(api_content)
-        media = info['extended_entities']['media']
-
-    for medium in media:
-        if 'video_info' in medium:
-            variants = medium['video_info']['variants']
-            variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
-            title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0].split('.')[0]
-            urls = [ variants[-1]['url'] ]
+    if 'photos' in info:
+        for photo in info['photos']:
+            photo_url = photo['url']
+            title = item_id + '_' + photo_url.split('.')[-2].split('/')[-1]
+            urls = [ photo_url + ':orig' ]
             size = urls_size(urls)
-            mime, ext = variants[-1]['content_type'], 'mp4'
+            ext = photo_url.split('.')[-1]
 
-            print_info(site_info, title, mime, size)
+            print_info(site_info, title, ext, size)
             if not info_only:
                 download_urls(urls, title, ext, size, output_dir, merge=merge)
 
-        else:
-            title = item_id + '_' + medium['media_url_https'].split('.')[-2].split('/')[-1]
-            urls = [ medium['media_url_https'] + ':orig' ]
+    if 'video' in info:
+        for mediaDetail in info['mediaDetails']:
+            variants = mediaDetail['video_info']['variants']
+            variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
+            title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0].split('.')[0]
+            urls = [ variants[-1]['url'] ]
             size = urls_size(urls)
-            ext = medium['media_url_https'].split('.')[-1]
+            mime, ext = variants[-1]['content_type'], 'mp4'
 
             print_info(site_info, title, ext, size)
             if not info_only:
                 download_urls(urls, title, ext, size, output_dir, merge=merge)
 
+    # TODO: should we deal with quoted tweets?
+
 
 site_info = "Twitter.com"
 download = twitter_download

From f9cbdc2656bcca7edabd90fa75b501dc7b52be32 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 11 Jul 2023 17:20:12 +0200
Subject: [PATCH 1151/1225] [twitter] minor fix

---
 src/you_get/extractors/twitter.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 43cfa6a4b1..4a439fe85f 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -62,6 +62,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
 
     if 'video' in info:
         for mediaDetail in info['mediaDetails']:
+            if 'video_info' not in mediaDetail: continue
             variants = mediaDetail['video_info']['variants']
             variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
             title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0].split('.')[0]

From 9f38d7d76f2df34fa1bd72b826c5248a3aba67d3 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 25 Jul 2023 23:42:57 +0200
Subject: [PATCH 1152/1225] [common] update UA

---
 src/you_get/common.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index bdb67bac77..4095dc5243 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -141,11 +141,11 @@
 prefix = None
 
 fake_headers = {
-    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',  # noqa
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.74 Safari/537.36 Edg/79.0.309.43',  # noqa
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.183'  # Latest Edge
 }
 
 if sys.stdout.isatty():

From 30d6c642f94d19b979cc4ea3461db1fea1901a6b Mon Sep 17 00:00:00 2001
From: OneCloud <linzhanyu@gmail.com>
Date: Sat, 16 Dec 2023 23:17:22 +0800
Subject: [PATCH 1153/1225] [bilibili] api url update.

---
 src/you_get/extractors/bilibili.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 6335e6dd87..b4e241c567 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -100,7 +100,8 @@ def bilibili_interface_api(cid, qn=0):
         appkey, sec = ''.join([chr(ord(i) + 2) for i in entropy[::-1]]).split(':')
         params = 'appkey=%s&cid=%s&otype=json&qn=%s&quality=%s&type=' % (appkey, cid, qn, qn)
         chksum = hashlib.md5(bytes(params + sec, 'utf8')).hexdigest()
-        return 'https://interface.bilibili.com/v2/playurl?%s&sign=%s' % (params, chksum)
+        return 'https://api.bilibili.com/x/player/wbi/v2?%s&sign=%s' % (params, chksum)
+
 
     @staticmethod
     def bilibili_live_api(cid):

From ac01a66b62c15d27b6a3b73257865dd06ad04433 Mon Sep 17 00:00:00 2001
From: Johnny <hellojinqiang@gmail.com>
Date: Mon, 18 Dec 2023 17:28:54 +0800
Subject: [PATCH 1154/1225] [twitter] fix twitter video download

---
 src/you_get/extractors/twitter.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 4a439fe85f..d995fc5842 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -40,7 +40,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     page_title = "{} [{}]".format(screen_name, item_id)
 
     # FIXME: this API won't work for protected or nsfw contents
-    api_url = 'https://cdn.syndication.twimg.com/tweet-result?id=%s' % item_id
+    api_url = 'https://cdn.syndication.twimg.com/tweet-result?id=%s&token=!' % item_id
     content = get_content(api_url)
     info = json.loads(content)
 

From c7e5a297478f5207af0fa0ba8a3c76ccda33b200 Mon Sep 17 00:00:00 2001
From: ifui <ifui@foxmail.com>
Date: Tue, 9 Jan 2024 09:53:41 +0800
Subject: [PATCH 1155/1225] fix ffmpeg: Unrecognized option 'absf'.

---
 src/you_get/processor/ffmpeg.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/processor/ffmpeg.py b/src/you_get/processor/ffmpeg.py
index 50e2c9fefb..e8639e89d9 100755
--- a/src/you_get/processor/ffmpeg.py
+++ b/src/you_get/processor/ffmpeg.py
@@ -175,7 +175,7 @@ def ffmpeg_concat_flv_to_mp4(files, output='output.mp4'):
     if FFMPEG == 'avconv':
         params += ['-c', 'copy']
     else:
-        params += ['-c', 'copy', '-absf', 'aac_adtstoasc']
+        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
     params.extend(['--', output])
 
     if subprocess.call(params, stdin=STDIN) == 0:
@@ -229,7 +229,7 @@ def ffmpeg_concat_mp4_to_mp4(files, output='output.mp4'):
     if FFMPEG == 'avconv':
         params += ['-c', 'copy']
     else:
-        params += ['-c', 'copy', '-absf', 'aac_adtstoasc']
+        params += ['-c', 'copy', '-bsf:a', 'aac_adtstoasc']
     params.extend(['--', output])
 
     subprocess.check_call(params, stdin=STDIN)

From 6d577712489d749b9e6401c3ba07c9e263c34dd8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Mar 2024 03:15:47 +0100
Subject: [PATCH 1156/1225] [imgur] fix extraction

---
 src/you_get/extractors/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index d612a30a88..14fd7f1a16 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -54,7 +54,7 @@ def prepare(self, **kwargs):
             content = get_content(self.url)
             url = match1(content, r'meta property="og:video"[^>]+(https?://i.imgur.com/[^"?]+)') or \
                 match1(content, r'meta property="og:image"[^>]+(https?://i.imgur.com/[^"?]+)')
-            _, container, size = url_info(url)
+            _, container, size = url_info(url, faker=True)
             self.streams = {
                 'original': {
                     'src': [url],

From 0b7a91fc064ebcd497d53a7c9a302ba7e74858fe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 4 Mar 2024 03:16:43 +0100
Subject: [PATCH 1157/1225] [imgur] fix extraction

---
 src/you_get/extractors/imgur.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 14fd7f1a16..09395202ae 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -39,7 +39,7 @@ def prepare(self, **kwargs):
 
         elif re.search(r'i\.imgur\.com/', self.url):
             # direct image
-            _, container, size = url_info(self.url)
+            _, container, size = url_info(self.url, faker=True)
             self.streams = {
                 'original': {
                     'src': [self.url],

From 01fa32419ea78f9f76dfeb8ced96a1ab3166b0e0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 18 Apr 2024 22:54:01 +0200
Subject: [PATCH 1158/1225] [common] update UA

---
 src/you_get/common.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 4095dc5243..ad3d327885 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -145,7 +145,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/115.0.1901.183'  # Latest Edge
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/123.0.2420.97'  # Latest Edge
 }
 
 if sys.stdout.isatty():
@@ -352,6 +352,7 @@ def getHttps(host, url, headers, debuglevel=0):
     conn.set_debuglevel(debuglevel)
     conn.request("GET", url, headers=headers)
     resp = conn.getresponse()
+    logging.debug('getHttps: %s' % resp.getheaders())
     set_cookie = resp.getheader('set-cookie')
 
     data = resp.read()
@@ -362,7 +363,7 @@ def getHttps(host, url, headers, debuglevel=0):
         pass
 
     conn.close()
-    return str(data, encoding='utf-8'), set_cookie
+    return str(data, encoding='utf-8'), set_cookie  # TODO: support raw data
 
 
 # DEPRECATED in favor of get_content()

From 1c1f9828698cecf3421d08adf44ae21c1514dbec Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Thu, 9 May 2024 17:18:11 +0800
Subject: [PATCH 1159/1225] The installation instructions failed for two
 reasons: 1. python 3.12 has removed the `imp` module and has replaced it by
 the `importlib` module. [1](https://docs.python.org/3/whatsnew/3.12.html#imp)
 2. the use of setup.py as in the command `python setup.py install` is
 deprecated and should be replaced by the recommended way of `python -m pip
 install path/to/project`.
 [2](https://packaging.python.org/en/latest/discussions/setup-py-deprecated/)

---
 README.md | 10 ++++++----
 setup.py  | 17 +++++++++++++++--
 2 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 44c102cde8..588ce309d6 100644
--- a/README.md
+++ b/README.md
@@ -80,16 +80,18 @@ You may either download the [stable](https://github.com/soimort/you-get/archive/
 Alternatively, run
 
 ```
-$ [sudo] python3 setup.py install
+$ cd path/to/you-get
+$ [sudo] python -m pip install .
 ```
 
 Or
 
 ```
-$ python3 setup.py install --user
+$ cd path/to/you-get
+$ python -m pip install . --user
 ```
 
-to install `you-get` to a permanent path.
+to install `you-get` to a permanent path. (And don't omit the dot `.` representing the current directory)
 
 You can also use the [pipenv](https://pipenv.pypa.io/en/latest) to install the `you-get` in the Python virtual environment.
 
@@ -107,7 +109,7 @@ This is the recommended way for all developers, even if you don't often code in
 $ git clone git://github.com/soimort/you-get.git
 ```
 
-Then put the cloned directory into your `PATH`, or run `./setup.py install` to install `you-get` to a permanent path.
+Then put the cloned directory into your `PATH`, or run `python -m pip install path/to/you-get` to install `you-get` to a permanent path.
 
 ### Option 5: Homebrew (Mac only)
 
diff --git a/setup.py b/setup.py
index 24dc9fb271..470c99ed4b 100755
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,20 @@
 
 PROJ_METADATA = '%s.json' % PROJ_NAME
 
-import os, json, imp
+import importlib.util
+import importlib.machinery
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
+
+import os, json
 here = os.path.abspath(os.path.dirname(__file__))
 proj_info = json.loads(open(os.path.join(here, PROJ_METADATA), encoding='utf-8').read())
 try:
@@ -13,7 +26,7 @@
 except:
     README = ""
 CHANGELOG = open(os.path.join(here, 'CHANGELOG.rst'), encoding='utf-8').read()
-VERSION = imp.load_source('version', os.path.join(here, 'src/%s/version.py' % PACKAGE_NAME)).__version__
+VERSION = load_source('version', os.path.join(here, 'src/%s/version.py' % PACKAGE_NAME)).__version__
 
 from setuptools import setup, find_packages
 setup(

From f7face20fa0e8b566c70141b123b62777b9a98cf Mon Sep 17 00:00:00 2001
From: wyzypa <wyzypa@gmail.com>
Date: Fri, 10 May 2024 10:29:44 +0800
Subject: [PATCH 1160/1225] fix: [extractors.bilibili] add headers when
 requesting for danmaku

---
 src/you_get/extractors/bilibili.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index b4e241c567..9860d5d6ec 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -335,7 +335,7 @@ def prepare(self, **kwargs):
                                                             'src': [[baseurl]], 'size': size}
 
             # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
         # bangumi
         elif sort == 'bangumi':
@@ -414,7 +414,7 @@ def prepare(self, **kwargs):
                                                         'src': [[baseurl], [audio_baseurl]], 'size': size}
 
             # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
         # vc video
         elif sort == 'vc':
@@ -596,7 +596,7 @@ def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
                                                         'src': [[baseurl]], 'size': size}
 
         # get danmaku
-        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid)
+        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
     def extract(self, **kwargs):
         # set UA and referer for downloading

From a4d34ff6b1821ce9c6f1f05423776c54f29dedac Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 17 May 2024 18:19:12 +0200
Subject: [PATCH 1161/1225] [bilibili] change all http to https

---
 src/you_get/extractors/bilibili.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/bilibili.py b/src/you_get/extractors/bilibili.py
index 9860d5d6ec..ea67f92fcf 100644
--- a/src/you_get/extractors/bilibili.py
+++ b/src/you_get/extractors/bilibili.py
@@ -335,7 +335,7 @@ def prepare(self, **kwargs):
                                                             'src': [[baseurl]], 'size': size}
 
             # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
+            self.danmaku = get_content('https://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
         # bangumi
         elif sort == 'bangumi':
@@ -414,7 +414,7 @@ def prepare(self, **kwargs):
                                                         'src': [[baseurl], [audio_baseurl]], 'size': size}
 
             # get danmaku
-            self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
+            self.danmaku = get_content('https://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
         # vc video
         elif sort == 'vc':
@@ -596,7 +596,7 @@ def prepare_by_cid(self,avid,cid,title,html_content,playinfo,playinfo_,url):
                                                         'src': [[baseurl]], 'size': size}
 
         # get danmaku
-        self.danmaku = get_content('http://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
+        self.danmaku = get_content('https://comment.bilibili.com/%s.xml' % cid, headers=self.bilibili_headers(referer=self.url))
 
     def extract(self, **kwargs):
         # set UA and referer for downloading

From 57f6502e801c493feddcb5ab84ae155ead515886 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 17 May 2024 20:57:37 +0200
Subject: [PATCH 1162/1225] [twitter] fix extraction (x.com)

---
 README.md                         | 2 +-
 src/you_get/common.py             | 8 ++++++--
 src/you_get/extractors/twitter.py | 6 +++---
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 588ce309d6..9127e57d4d 100644
--- a/README.md
+++ b/README.md
@@ -376,7 +376,7 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | Site | URL | Videos? | Images? | Audios? |
 | :--: | :-- | :-----: | :-----: | :-----: |
 | **YouTube** | <https://www.youtube.com/>    |✓| | |
-| **Twitter** | <https://twitter.com/>        |✓|✓| |
+| **X (Twitter)** | <https://x.com/>        |✓|✓| |
 | VK          | <http://vk.com/>              |✓|✓| |
 | Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
diff --git a/src/you_get/common.py b/src/you_get/common.py
index ad3d327885..d14eb980db 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -113,6 +113,7 @@
     'veoh'             : 'veoh',
     'vine'             : 'vine',
     'vk'               : 'vk',
+    'x'                : 'twitter',
     'xiaokaxiu'        : 'yixia',
     'xiaojiadianvideo' : 'fc2video',
     'ximalaya'         : 'ximalaya',
@@ -1856,9 +1857,12 @@ def url_to_module(url):
         )
     else:
         try:
-            location = get_location(url) # t.co isn't happy with fake_headers
+            try:
+                location = get_location(url) # t.co isn't happy with fake_headers
+            except:
+                location = get_location(url, headers=fake_headers)
         except:
-            location = get_location(url, headers=fake_headers)
+            location = get_location(url, headers=fake_headers, get_method='GET')
 
         if location and location != url and not location.startswith('/'):
             return url_to_module(location)
diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index d995fc5842..b0125c679c 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -34,9 +34,9 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    m = re.match('^https?://(mobile\.)?twitter\.com/([^/]+)/status/(\d+)', url)
+    m = re.match('^https?://(mobile\.)?(x|twitter)\.com/([^/]+)/status/(\d+)', url)
     assert m
-    screen_name, item_id = m.group(2), m.group(3)
+    screen_name, item_id = m.group(3), m.group(4)
     page_title = "{} [{}]".format(screen_name, item_id)
 
     # FIXME: this API won't work for protected or nsfw contents
@@ -77,6 +77,6 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
     # TODO: should we deal with quoted tweets?
 
 
-site_info = "Twitter.com"
+site_info = "X.com"
 download = twitter_download
 download_playlist = playlist_not_supported('twitter')

From 873ffdb61eb461b4c71ab7e12151864b49a23f3a Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Sun, 19 May 2024 00:58:08 +0800
Subject: [PATCH 1163/1225] Update setup.py: compatibility for older python
 versions

---
 setup.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/setup.py b/setup.py
index 470c99ed4b..ea9846710e 100755
--- a/setup.py
+++ b/setup.py
@@ -5,18 +5,21 @@
 
 PROJ_METADATA = '%s.json' % PROJ_NAME
 
-import importlib.util
-import importlib.machinery
-
-def load_source(modname, filename):
-    loader = importlib.machinery.SourceFileLoader(modname, filename)
-    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
-    module = importlib.util.module_from_spec(spec)
-    # The module is always executed and not cached in sys.modules.
-    # Uncomment the following line to cache the module.
-    # sys.modules[module.__name__] = module
-    loader.exec_module(module)
-    return module
+import sys
+if (sys.version_info >= (3, 12)):
+    import importlib.util
+    import importlib.machinery
+    def load_source(modname, filename):
+        loader = importlib.machinery.SourceFileLoader(modname, filename)
+        spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+        module = importlib.util.module_from_spec(spec)
+        # The module is always executed and not cached in sys.modules.
+        # Uncomment the following line to cache the module.
+        # sys.modules[module.__name__] = module
+        loader.exec_module(module)
+        return module
+else:
+    from imp import load_source
 
 import os, json
 here = os.path.abspath(os.path.dirname(__file__))

From 317cc467e7f738390a9d451ad530736d0e848690 Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Sun, 19 May 2024 01:01:09 +0800
Subject: [PATCH 1164/1225] Update python-package.yml: add python-version 3.12
 job

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 39793c03f7..51d56c914b 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', pypy-3.8, pypy-3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9]
 
     steps:
     - uses: actions/checkout@v3

From adeaeb896775c5dd6397fdd6eb98a2750aa99717 Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Sun, 19 May 2024 01:03:54 +0800
Subject: [PATCH 1165/1225] Update you-get.json: add info Python :: 3.11 and
 3.12

---
 you-get.json | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/you-get.json b/you-get.json
index bb94ba0041..adf604dc5c 100644
--- a/you-get.json
+++ b/you-get.json
@@ -22,6 +22,8 @@
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Internet",
     "Topic :: Internet :: WWW/HTTP",
     "Topic :: Multimedia",

From b935d3ed02ea029f6c220e7123fd321e489fc7d8 Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Sun, 19 May 2024 02:01:31 +0800
Subject: [PATCH 1166/1225] Update python-package.yml: update setuptools

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 51d56c914b..98b6c7de3c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -25,7 +25,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
+        python -m pip install --upgrade pip setuptools
         pip install flake8 pytest
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8

From 19a4f15d6014c66c99d48be4a024b89407fe428a Mon Sep 17 00:00:00 2001
From: cerenkov <cerenkov@qq.com>
Date: Sun, 19 May 2024 12:03:54 +0800
Subject: [PATCH 1167/1225] Revert "Update setup.py: compatibility for older
 python versions"

This reverts commit 873ffdb61eb461b4c71ab7e12151864b49a23f3a.

Minimun python version 3.7 already assures importlib availability
---
 setup.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/setup.py b/setup.py
index ea9846710e..470c99ed4b 100755
--- a/setup.py
+++ b/setup.py
@@ -5,21 +5,18 @@
 
 PROJ_METADATA = '%s.json' % PROJ_NAME
 
-import sys
-if (sys.version_info >= (3, 12)):
-    import importlib.util
-    import importlib.machinery
-    def load_source(modname, filename):
-        loader = importlib.machinery.SourceFileLoader(modname, filename)
-        spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
-        module = importlib.util.module_from_spec(spec)
-        # The module is always executed and not cached in sys.modules.
-        # Uncomment the following line to cache the module.
-        # sys.modules[module.__name__] = module
-        loader.exec_module(module)
-        return module
-else:
-    from imp import load_source
+import importlib.util
+import importlib.machinery
+
+def load_source(modname, filename):
+    loader = importlib.machinery.SourceFileLoader(modname, filename)
+    spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+    module = importlib.util.module_from_spec(spec)
+    # The module is always executed and not cached in sys.modules.
+    # Uncomment the following line to cache the module.
+    # sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module
 
 import os, json
 here = os.path.abspath(os.path.dirname(__file__))

From 7f8ebe1c93b1f35545989445369ab4f19dc34af8 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 20 May 2024 04:18:10 +0200
Subject: [PATCH 1168/1225] [tests] disable test_tiktok temporarily

---
 tests/test.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index c0f3836aed..a8c7023d0f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -55,10 +55,10 @@ def test_acfun(self):
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
         #)
 
-    def test_tiktok(self):
-        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
-        tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
-        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
+    #def test_tiktok(self):
+        #tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+        #tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
+        #tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
 
     def test_twitter(self):
         twitter.download('https://twitter.com/elonmusk/status/1530516552084234244', info_only=True)

From 0b9fec525152ef1c705864245970925c64a65872 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 20 May 2024 04:56:36 +0200
Subject: [PATCH 1169/1225] [imgur] fix extraction

---
 src/you_get/extractors/imgur.py | 8 +++++---
 tests/test.py                   | 1 +
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/imgur.py b/src/you_get/extractors/imgur.py
index 09395202ae..2726c97439 100644
--- a/src/you_get/extractors/imgur.py
+++ b/src/you_get/extractors/imgur.py
@@ -13,9 +13,11 @@ class Imgur(VideoExtractor):
     ]
 
     def prepare(self, **kwargs):
+        self.ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/123.0.2420.97'
+
         if re.search(r'imgur\.com/a/', self.url):
             # album
-            content = get_content(self.url)
+            content = get_content(self.url, headers=fake_headers)
             album = match1(content, r'album\s*:\s*({.*}),') or \
                     match1(content, r'image\s*:\s*({.*}),')
             album = json.loads(album)
@@ -51,10 +53,10 @@ def prepare(self, **kwargs):
 
         else:
             # gallery image
-            content = get_content(self.url)
+            content = get_content(self.url, headers=fake_headers)
             url = match1(content, r'meta property="og:video"[^>]+(https?://i.imgur.com/[^"?]+)') or \
                 match1(content, r'meta property="og:image"[^>]+(https?://i.imgur.com/[^"?]+)')
-            _, container, size = url_info(url, faker=True)
+            _, container, size = url_info(url, headers={'User-Agent': fake_headers['User-Agent']})
             self.streams = {
                 'original': {
                     'src': [url],
diff --git a/tests/test.py b/tests/test.py
index a8c7023d0f..d3cd099d4b 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -19,6 +19,7 @@
 class YouGetTests(unittest.TestCase):
     def test_imgur(self):
         imgur.download('http://imgur.com/WVLk5nD', info_only=True)
+        imgur.download('https://imgur.com/we-should-have-listened-WVLk5nD', info_only=True)
 
     def test_magisto(self):
         magisto.download(

From 97bb0a6d4e06ddc679ca4176356e99c0aa532566 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 21 May 2024 01:45:12 +0200
Subject: [PATCH 1170/1225] python-package.yml: add pypy-3.10

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 98b6c7de3c..1d9afe9fa5 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9, pypy-3.10]
 
     steps:
     - uses: actions/checkout@v3

From bf3d82bd4b11e6ef06634c04d2bbb68593025984 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 21 May 2024 23:59:39 +0200
Subject: [PATCH 1171/1225] [tiktok] fix extraction

---
 src/you_get/extractors/tiktok.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 2c4892f629..3cbb59d347 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -27,12 +27,12 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     tt_chain_token = r1('tt_chain_token=([^;]+);', set_cookie)
     headers['Cookie'] = 'tt_chain_token=%s' % tt_chain_token
 
-    data = r1(r'window\[\'SIGI_STATE\'\]=(.*?);window\[\'SIGI_RETRY\'\]', html) or \
-        r1(r'<script id="SIGI_STATE" type="application/json">(.*?)</script>', html)
+    data = r1(r'<script id="__UNIVERSAL_DATA_FOR_REHYDRATION__" type="application/json">(.*?)</script>', html)
     info = json.loads(data)
-    downloadAddr = info['ItemModule'][vid]['video']['downloadAddr']
-    author = info['ItemModule'][vid]['author']  # same as uniqueId
-    nickname = info['UserModule']['users'][author]['nickname']
+    itemStruct = info['__DEFAULT_SCOPE__']['webapp.video-detail']['itemInfo']['itemStruct']
+    downloadAddr = itemStruct['video']['downloadAddr']
+    author = itemStruct['author']['uniqueId']
+    nickname = itemStruct['author']['nickname']
     title = '%s [%s]' % (nickname or author, vid)
 
     mime, ext, size = url_info(downloadAddr, headers=headers)

From 5e7a06f01de18fb02992e27ba088bd613a8a5055 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 00:01:35 +0200
Subject: [PATCH 1172/1225] [tests] enable test_tiktok, etc.

---
 tests/test.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index d3cd099d4b..63e10e1753 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -43,8 +43,8 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
 
-    #def test_bilibili(self):
-    #    bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
+    def test_bilibili(self):
+        bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 
     #def test_soundcloud(self):
         ## single song
@@ -56,10 +56,10 @@ def test_acfun(self):
         #    'https://soundcloud.com/anthony-flieger/sets/cytus', info_only=True
         #)
 
-    #def test_tiktok(self):
-        #tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
-        #tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
-        #tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
+    def test_tiktok(self):
+        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
+        tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
+        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
 
     def test_twitter(self):
         twitter.download('https://twitter.com/elonmusk/status/1530516552084234244', info_only=True)

From 290c74569902db2f1e58521e4e3bd7e1d47727a9 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 00:04:35 +0200
Subject: [PATCH 1173/1225] [tests] remove test_acfun (404 URL)

---
 tests/test.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 63e10e1753..78de65614f 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,9 +40,6 @@ def test_youtube(self):
         #    'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
         #)
 
-    def test_acfun(self):
-        acfun.download('https://www.acfun.cn/v/ac11701912', info_only=True)
-
     def test_bilibili(self):
         bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 

From 763f8a4e650b1982c3da09e0ea8b84c32d1a93de Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 00:05:57 +0200
Subject: [PATCH 1174/1225] [tests] update test_acfun

---
 tests/test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test.py b/tests/test.py
index 78de65614f..720dc8399c 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -40,6 +40,9 @@ def test_youtube(self):
         #    'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
         #)
 
+    def test_acfun(self):
+        acfun.download('https://www.acfun.cn/v/ac44560432', info_only=True)
+
     def test_bilibili(self):
         bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 

From 1899b3e4fad6ea77a98aa1a4acafc8c27e535a9d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 00:24:51 +0200
Subject: [PATCH 1175/1225] update README.md (pip3 -> pip)

---
 README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 9127e57d4d..c4ea4c0043 100644
--- a/README.md
+++ b/README.md
@@ -63,9 +63,9 @@ The following dependencies are recommended:
 
 ### Option 1: Install via pip
 
-The official release of `you-get` is distributed on [PyPI](https://pypi.python.org/pypi/you-get), and can be installed easily from a PyPI mirror via the [pip](https://en.wikipedia.org/wiki/Pip_\(package_manager\)) package manager. Note that you must use the Python 3 version of `pip`:
+The official release of `you-get` is distributed on [PyPI](https://pypi.python.org/pypi/you-get), and can be installed easily from a PyPI mirror via the [pip](https://en.wikipedia.org/wiki/Pip_\(package_manager\)) package manager: (Note that you must use the Python 3 version of `pip`)
 
-    $ pip3 install you-get
+    $ pip install you-get
 
 ### Option 2: Install via [Antigen](https://github.com/zsh-users/antigen) (for Zsh users)
 
@@ -136,7 +136,7 @@ Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completi
 Based on which option you chose to install `you-get`, you may upgrade it via:
 
 ```
-$ pip3 install --upgrade you-get
+$ pip install --upgrade you-get
 ```
 
 or download the latest release via:
@@ -148,7 +148,7 @@ $ you-get https://github.com/soimort/you-get/archive/master.zip
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
 
 ```
-$ pip3 install --upgrade git+https://github.com/soimort/you-get@develop
+$ pip install --upgrade git+https://github.com/soimort/you-get@develop
 ```
 
 ## Getting Started

From 34e4c8651bb2ce752e2320a15262bf1bddad6b0e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:02:27 +0200
Subject: [PATCH 1176/1225] [tumblr] print info for each pic

---
 src/you_get/extractors/tumblr.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 1fdfcad06f..5b5c22d681 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -82,16 +82,16 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             except: pass
 
         if tuggles:
-            size = sum([tuggles[t]['size'] for t in tuggles])
-            print_info(site_info, page_title, None, size)
-
-            if not info_only:
-                for t in tuggles:
-                    title = tuggles[t]['title']
-                    ext = tuggles[t]['ext']
-                    size = tuggles[t]['size']
-                    url = tuggles[t]['url']
-                    print_info(site_info, title, ext, size)
+            #size = sum([tuggles[t]['size'] for t in tuggles])
+            #print_info(site_info, page_title, None, size)
+
+            for t in tuggles:
+                title = tuggles[t]['title']
+                ext = tuggles[t]['ext']
+                size = tuggles[t]['size']
+                url = tuggles[t]['url']
+                print_info(site_info, title, ext, size)
+                if not info_only:
                     download_urls([url], title, ext, size,
                                   output_dir=output_dir)
             return

From f6fd3fb867e9ce9a9219f9f0c6bf091797074a9d Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:04:21 +0200
Subject: [PATCH 1177/1225] [tumblr] add prefix

---
 src/you_get/extractors/tumblr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 5b5c22d681..b0dc99f6ad 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -86,7 +86,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             #print_info(site_info, page_title, None, size)
 
             for t in tuggles:
-                title = tuggles[t]['title']
+                title = '[tumblr] ' + tuggles[t]['title']
                 ext = tuggles[t]['ext']
                 size = tuggles[t]['size']
                 url = tuggles[t]['url']

From 1a3bcb462aef6addd659c0d0df4f242b61ab6f4c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:10:56 +0200
Subject: [PATCH 1178/1225] update README.md (http:// -> https://)

---
 README.md | 133 ++++++++++++++++++++++++++----------------------------
 1 file changed, 64 insertions(+), 69 deletions(-)

diff --git a/README.md b/README.md
index c4ea4c0043..5748775ecd 100644
--- a/README.md
+++ b/README.md
@@ -268,25 +268,20 @@ Type:       JPEG Image (image/jpeg)
 Size:       0.06 MiB (66482 Bytes)
 
 Downloading rms.jpg ...
-100.0% (  0.1/0.1  MB) ├████████████████████████████████████████┤[1/1]  127 kB/s
+ 100% (  0.1/  0.1MB) ├████████████████████████████████████████┤[1/1]  127 kB/s
 ```
 
 Otherwise, `you-get` will scrape the web page and try to figure out if there's anything interesting to you:
 
 ```
-$ you-get http://kopasas.tumblr.com/post/69361932517
+$ you-get https://kopasas.tumblr.com/post/69361932517
 Site:       Tumblr.com
-Title:      kopasas
-Type:       Unknown type (None)
-Size:       0.51 MiB (536583 Bytes)
-
-Site:       Tumblr.com
-Title:      tumblr_mxhg13jx4n1sftq6do1_1280
+Title:      [tumblr] tumblr_mxhg13jx4n1sftq6do1_640
 Type:       Portable Network Graphics (image/png)
-Size:       0.51 MiB (536583 Bytes)
+Size:       0.11 MiB (118484 Bytes)
 
-Downloading tumblr_mxhg13jx4n1sftq6do1_1280.png ...
-100.0% (  0.5/0.5  MB) ├████████████████████████████████████████┤[1/1]   22 MB/s
+Downloading [tumblr] tumblr_mxhg13jx4n1sftq6do1_640.png ...
+ 100% (  0.1/  0.1MB) ├████████████████████████████████████████┤[1/1]   22 MB/s
 ```
 
 **Note:**
@@ -377,81 +372,81 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | :--: | :-- | :-----: | :-----: | :-----: |
 | **YouTube** | <https://www.youtube.com/>    |✓| | |
 | **X (Twitter)** | <https://x.com/>        |✓|✓| |
-| VK          | <http://vk.com/>              |✓|✓| |
+| VK          | <https://vk.com/>              |✓|✓| |
 | Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
-| Veoh        | <http://www.veoh.com/>        |✓| | |
+| Veoh        | <https://www.veoh.com/>        |✓| | |
 | **Tumblr**  | <https://www.tumblr.com/>     |✓|✓|✓|
-| TED         | <http://www.ted.com/>         |✓| | |
+| TED         | <https://www.ted.com/>         |✓| | |
 | SoundCloud  | <https://soundcloud.com/>     | | |✓|
 | SHOWROOM    | <https://www.showroom-live.com/> |✓| | |
 | Pinterest   | <https://www.pinterest.com/>  | |✓| |
-| MTV81       | <http://www.mtv81.com/>       |✓| | |
+| MTV81       | <https://www.mtv81.com/>       |✓| | |
 | Mixcloud    | <https://www.mixcloud.com/>   | | |✓|
-| Metacafe    | <http://www.metacafe.com/>    |✓| | |
-| Magisto     | <http://www.magisto.com/>     |✓| | |
+| Metacafe    | <https://www.metacafe.com/>    |✓| | |
+| Magisto     | <https://www.magisto.com/>     |✓| | |
 | Khan Academy | <https://www.khanacademy.org/> |✓| | |
 | Internet Archive | <https://archive.org/>   |✓| | |
 | **Instagram** | <https://instagram.com/>    |✓|✓| |
-| InfoQ       | <http://www.infoq.com/presentations/> |✓| | |
-| Imgur       | <http://imgur.com/>           | |✓| |
-| Heavy Music Archive | <http://www.heavy-music.ru/> | | |✓|
-| Freesound   | <http://www.freesound.org/>   | | |✓|
+| InfoQ       | <https://www.infoq.com/presentations/> |✓| | |
+| Imgur       | <https://imgur.com/>           | |✓| |
+| Heavy Music Archive | <https://www.heavy-music.ru/> | | |✓|
+| Freesound   | <https://www.freesound.org/>   | | |✓|
 | Flickr      | <https://www.flickr.com/>     |✓|✓| |
-| FC2 Video   | <http://video.fc2.com/>       |✓| | |
+| FC2 Video   | <https://video.fc2.com/>       |✓| | |
 | Facebook    | <https://www.facebook.com/>   |✓| | |
-| eHow        | <http://www.ehow.com/>        |✓| | |
-| Dailymotion | <http://www.dailymotion.com/> |✓| | |
-| Coub        | <http://coub.com/>            |✓| | |
-| CBS         | <http://www.cbs.com/>         |✓| | |
-| Bandcamp    | <http://bandcamp.com/>        | | |✓|
-| AliveThai   | <http://alive.in.th/>         |✓| | |
-| interest.me | <http://ch.interest.me/tvn>   |✓| | |
-| **755<br/>ナナゴーゴー** | <http://7gogo.jp/> |✓|✓| |
-| **niconico<br/>ニコニコ動画** | <http://www.nicovideo.jp/> |✓| | |
-| **163<br/>网易视频<br/>网易云音乐** | <http://v.163.com/><br/><http://music.163.com/> |✓| |✓|
-| 56网     | <http://www.56.com/>           |✓| | |
-| **AcFun** | <http://www.acfun.cn/>        |✓| | |
-| **Baidu<br/>百度贴吧** | <http://tieba.baidu.com/> |✓|✓| |
-| 爆米花网 | <http://www.baomihua.com/>     |✓| | |
-| **bilibili<br/>哔哩哔哩** | <http://www.bilibili.com/> |✓|✓|✓|
-| 豆瓣     | <http://www.douban.com/>       |✓| |✓|
-| 斗鱼     | <http://www.douyutv.com/>      |✓| | |
-| 凤凰视频 | <http://v.ifeng.com/>          |✓| | |
-| 风行网   | <http://www.fun.tv/>           |✓| | |
-| iQIYI<br/>爱奇艺 | <http://www.iqiyi.com/> |✓| | |
-| 激动网   | <http://www.joy.cn/>           |✓| | |
-| 酷6网    | <http://www.ku6.com/>          |✓| | |
-| 酷狗音乐 | <http://www.kugou.com/>        | | |✓|
-| 酷我音乐 | <http://www.kuwo.cn/>          | | |✓|
-| 乐视网   | <http://www.le.com/>           |✓| | |
-| 荔枝FM   | <http://www.lizhi.fm/>         | | |✓|
-| 懒人听书 | <http://www.lrts.me/>          | | |✓|
-| 秒拍     | <http://www.miaopai.com/>      |✓| | |
-| MioMio弹幕网 | <http://www.miomio.tv/>    |✓| | |
-| MissEvan<br/>猫耳FM | <http://www.missevan.com/> | | |✓|
+| eHow        | <https://www.ehow.com/>        |✓| | |
+| Dailymotion | <https://www.dailymotion.com/> |✓| | |
+| Coub        | <https://coub.com/>            |✓| | |
+| CBS         | <https://www.cbs.com/>         |✓| | |
+| Bandcamp    | <https://bandcamp.com/>        | | |✓|
+| AliveThai   | <https://alive.in.th/>         |✓| | |
+| interest.me | <https://ch.interest.me/tvn>   |✓| | |
+| **755<br/>ナナゴーゴー** | <https://7gogo.jp/> |✓|✓| |
+| **niconico<br/>ニコニコ動画** | <https://www.nicovideo.jp/> |✓| | |
+| **163<br/>网易视频<br/>网易云音乐** | <https://v.163.com/><br/><https://music.163.com/> |✓| |✓|
+| 56网     | <https://www.56.com/>           |✓| | |
+| **AcFun** | <https://www.acfun.cn/>        |✓| | |
+| **Baidu<br/>百度贴吧** | <https://tieba.baidu.com/> |✓|✓| |
+| 爆米花网 | <https://www.baomihua.com/>     |✓| | |
+| **bilibili<br/>哔哩哔哩** | <https://www.bilibili.com/> |✓|✓|✓|
+| 豆瓣     | <https://www.douban.com/>       |✓| |✓|
+| 斗鱼     | <https://www.douyutv.com/>      |✓| | |
+| 凤凰视频 | <https://v.ifeng.com/>          |✓| | |
+| 风行网   | <https://www.fun.tv/>           |✓| | |
+| iQIYI<br/>爱奇艺 | <https://www.iqiyi.com/> |✓| | |
+| 激动网   | <https://www.joy.cn/>           |✓| | |
+| 酷6网    | <https://www.ku6.com/>          |✓| | |
+| 酷狗音乐 | <https://www.kugou.com/>        | | |✓|
+| 酷我音乐 | <https://www.kuwo.cn/>          | | |✓|
+| 乐视网   | <https://www.le.com/>           |✓| | |
+| 荔枝FM   | <https://www.lizhi.fm/>         | | |✓|
+| 懒人听书 | <https://www.lrts.me/>          | | |✓|
+| 秒拍     | <https://www.miaopai.com/>      |✓| | |
+| MioMio弹幕网 | <https://www.miomio.tv/>    |✓| | |
+| MissEvan<br/>猫耳FM | <https://www.missevan.com/> | | |✓|
 | 痞客邦   | <https://www.pixnet.net/>      |✓| | |
-| PPTV聚力 | <http://www.pptv.com/>         |✓| | |
-| 齐鲁网   | <http://v.iqilu.com/>          |✓| | |
-| QQ<br/>腾讯视频 | <http://v.qq.com/>      |✓| | |
-| 企鹅直播 | <http://live.qq.com/>          |✓| | |
-| Sina<br/>新浪视频<br/>微博秒拍视频 | <http://video.sina.com.cn/><br/><http://video.weibo.com/> |✓| | |
-| Sohu<br/>搜狐视频 | <http://tv.sohu.com/> |✓| | |
-| **Tudou<br/>土豆** | <http://www.tudou.com/> |✓| | |
-| 阳光卫视 | <http://www.isuntv.com/>       |✓| | |
-| **Youku<br/>优酷** | <http://www.youku.com/> |✓| | |
-| 战旗TV   | <http://www.zhanqi.tv/lives>   |✓| | |
-| 央视网   | <http://www.cntv.cn/>          |✓| | |
-| Naver<br/>네이버 | <http://tvcast.naver.com/>     |✓| | |
-| 芒果TV   | <http://www.mgtv.com/>         |✓| | |
-| 火猫TV   | <http://www.huomao.com/>       |✓| | |
-| 阳光宽频网 | <http://www.365yg.com/>      |✓| | |
+| PPTV聚力 | <https://www.pptv.com/>         |✓| | |
+| 齐鲁网   | <https://v.iqilu.com/>          |✓| | |
+| QQ<br/>腾讯视频 | <https://v.qq.com/>      |✓| | |
+| 企鹅直播 | <https://live.qq.com/>          |✓| | |
+| Sina<br/>新浪视频<br/>微博秒拍视频 | <https://video.sina.com.cn/><br/><https://video.weibo.com/> |✓| | |
+| Sohu<br/>搜狐视频 | <https://tv.sohu.com/> |✓| | |
+| **Tudou<br/>土豆** | <https://www.tudou.com/> |✓| | |
+| 阳光卫视 | <https://www.isuntv.com/>       |✓| | |
+| **Youku<br/>优酷** | <https://www.youku.com/> |✓| | |
+| 战旗TV   | <https://www.zhanqi.tv/lives>   |✓| | |
+| 央视网   | <https://www.cntv.cn/>          |✓| | |
+| Naver<br/>네이버 | <https://tvcast.naver.com/>     |✓| | |
+| 芒果TV   | <https://www.mgtv.com/>         |✓| | |
+| 火猫TV   | <https://www.huomao.com/>       |✓| | |
+| 阳光宽频网 | <https://www.365yg.com/>      |✓| | |
 | 西瓜视频 | <https://www.ixigua.com/>      |✓| | |
 | 新片场 | <https://www.xinpianchang.com/>      |✓| | |
 | 快手 | <https://www.kuaishou.com/>      |✓|✓| |
 | 抖音 | <https://www.douyin.com/>      |✓| | |
 | TikTok | <https://www.tiktok.com/>      |✓| | |
-| 中国体育(TV) | <http://v.zhibo.tv/> </br><http://video.zhibo.tv/>    |✓| | |
+| 中国体育(TV) | <https://v.zhibo.tv/> </br><https://video.zhibo.tv/>    |✓| | |
 | 知乎 | <https://www.zhihu.com/>      |✓| | |
 
 For all other sites not on the list, the universal extractor will take care of finding and downloading interesting resources from the page.
@@ -464,7 +459,7 @@ Check if it's already a known problem on <https://github.com/soimort/you-get/wik
 
 ## Getting Involved
 
-You can reach us on the Gitter channel [#soimort/you-get](https://gitter.im/soimort/you-get) (here's how you [set up your IRC client](http://irc.gitter.im) for Gitter). If you have a quick question regarding `you-get`, ask it there.
+You can reach us on the Gitter channel [#soimort/you-get](https://gitter.im/soimort/you-get) (here's how you [set up your IRC client](https://irc.gitter.im) for Gitter). If you have a quick question regarding `you-get`, ask it there.
 
 If you are seeking to report an issue or contribute, please make sure to read [the guidelines](https://github.com/soimort/you-get/blob/develop/CONTRIBUTING.md) first.
 

From 00f6c01e1863168fa007d7698ff61d7c99552072 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:15:49 +0200
Subject: [PATCH 1179/1225] [vine] remove support (discontinued)

---
 README.md                          |  1 -
 src/you_get/common.py              |  1 -
 src/you_get/extractors/__init__.py |  1 -
 src/you_get/extractors/tumblr.py   |  4 ----
 src/you_get/extractors/twitter.py  |  1 -
 src/you_get/extractors/vine.py     | 36 ------------------------------
 6 files changed, 44 deletions(-)
 delete mode 100644 src/you_get/extractors/vine.py

diff --git a/README.md b/README.md
index 5748775ecd..0db87791ed 100644
--- a/README.md
+++ b/README.md
@@ -373,7 +373,6 @@ Use `--url`/`-u` to get a list of downloadable resource URLs extracted from the
 | **YouTube** | <https://www.youtube.com/>    |✓| | |
 | **X (Twitter)** | <https://x.com/>        |✓|✓| |
 | VK          | <https://vk.com/>              |✓|✓| |
-| Vine        | <https://vine.co/>            |✓| | |
 | Vimeo       | <https://vimeo.com/>          |✓| | |
 | Veoh        | <https://www.veoh.com/>        |✓| | |
 | **Tumblr**  | <https://www.tumblr.com/>     |✓|✓|✓|
diff --git a/src/you_get/common.py b/src/you_get/common.py
index d14eb980db..752c63f2ae 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -111,7 +111,6 @@
     'wanmen'           : 'wanmen',
     'weibo'            : 'miaopai',
     'veoh'             : 'veoh',
-    'vine'             : 'vine',
     'vk'               : 'vk',
     'x'                : 'twitter',
     'xiaokaxiu'        : 'yixia',
diff --git a/src/you_get/extractors/__init__.py b/src/you_get/extractors/__init__.py
index 8c43a8bca3..e68cd174f4 100755
--- a/src/you_get/extractors/__init__.py
+++ b/src/you_get/extractors/__init__.py
@@ -74,7 +74,6 @@
 from .ucas import *
 from .veoh import *
 from .vimeo import *
-from .vine import *
 from .vk import *
 from .w56 import *
 from .wanmen import *
diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index b0dc99f6ad..08e605e82c 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -6,7 +6,6 @@
 from .universal import *
 from .dailymotion import dailymotion_download
 from .vimeo import vimeo_download
-from .vine import vine_download
 
 def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     if re.match(r'https?://\d+\.media\.tumblr\.com/', url):
@@ -125,9 +124,6 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
             elif re.search(r'dailymotion\.com', iframe_url):
                 dailymotion_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
                 return
-            elif re.search(r'vine\.co', iframe_url):
-                vine_download(iframe_url, output_dir, merge=merge, info_only=info_only, **kwargs)
-                return
             else:
                 iframe_html = get_content(iframe_url)
                 real_url = r1(r'<source src="([^"]*)"', iframe_html)
diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index b0125c679c..b2c5878a9d 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -4,7 +4,6 @@
 
 from ..common import *
 from .universal import *
-from .vine import vine_download
 
 def extract_m3u(source):
     r1 = get_content(source)
diff --git a/src/you_get/extractors/vine.py b/src/you_get/extractors/vine.py
deleted file mode 100644
index d75454cf54..0000000000
--- a/src/you_get/extractors/vine.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-
-__all__ = ['vine_download']
-
-from ..common import *
-import json
-
-
-def vine_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
-    html = get_content(url)
-
-    video_id = r1(r'vine.co/v/([^/]+)', url)
-    title = r1(r'<title>([^<]*)</title>', html)
-    stream = r1(r'<meta property="twitter:player:stream" content="([^"]*)">', html)
-    if not stream:  # https://vine.co/v/.../card
-        stream = r1(r'"videoUrl":"([^"]+)"', html)
-        if stream:
-            stream = stream.replace('\\/', '/')
-        else:
-            posts_url = 'https://archive.vine.co/posts/' + video_id + '.json'
-            json_data = json.loads(get_content(posts_url))
-            stream = json_data['videoDashUrl']
-            title = json_data['description']
-            if title == "":
-                title = json_data['username'].replace(" ", "_") + "_" + video_id
-
-    mime, ext, size = url_info(stream)
-
-    print_info(site_info, title, mime, size)
-    if not info_only:
-        download_urls([stream], title, ext, size, output_dir, merge=merge)
-
-
-site_info = "Vine.co"
-download = vine_download
-download_playlist = playlist_not_supported('vine')

From 9aa538a403452901ccacca02deb12977f9cbcda0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:32:38 +0200
Subject: [PATCH 1180/1225] update LICENSE.txt

---
 LICENSE.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE.txt b/LICENSE.txt
index a193d8e254..fcc26433a4 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2012-2020 Mort Yao <mort.yao@gmail.com> and other contributors
+Copyright (c) 2012-2024 Mort Yao <mort.yao@gmail.com> and other contributors
               (https://github.com/soimort/you-get/graphs/contributors)
 Copyright (c) 2012 Boyu Guo <iambus@gmail.com>
 

From dbb767f9288b73d852945a1dc965fd12640c73fe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 01:58:47 +0200
Subject: [PATCH 1181/1225] version 0.4.1700

---
 SECURITY.md            | 2 +-
 src/you_get/version.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 5041b2f202..d9fb8cf340 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -2,4 +2,4 @@
 
 ## Reporting a Vulnerability
 
-Please report security issues to <email>
+Please report security issues to <mort.yao+you-get@gmail.com>.
diff --git a/src/you_get/version.py b/src/you_get/version.py
index 440488a9ac..14ed6a9977 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1650'
+__version__ = '0.4.1700'

From 29f513821df4b0ec3ed0b514a0897f8c336b51e7 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 22 May 2024 02:53:12 +0200
Subject: [PATCH 1182/1225] [tests] disable test_bilibili temporarily

---
 tests/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index 720dc8399c..e8a378f528 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -43,8 +43,8 @@ def test_youtube(self):
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac44560432', info_only=True)
 
-    def test_bilibili(self):
-        bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
+    #def test_bilibili(self):
+        #bilibili.download('https://www.bilibili.com/video/BV1sL4y177sC', info_only=True)
 
     #def test_soundcloud(self):
         ## single song

From 567d1059fce22fe790e059af9812ab7cd12135db Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 10:17:02 +0200
Subject: [PATCH 1183/1225] [youtube] fix 403 error and throttling (close
 #2950)

---
 setup.py                          |   3 +-
 src/you_get/extractors/youtube.py | 174 ++++++++++++++----------------
 tests/test.py                     |   6 +-
 3 files changed, 87 insertions(+), 96 deletions(-)

diff --git a/setup.py b/setup.py
index 470c99ed4b..0804ae33ec 100755
--- a/setup.py
+++ b/setup.py
@@ -56,7 +56,8 @@ def load_source(modname, filename):
 
     entry_points = {'console_scripts': proj_info['console_scripts']},
 
-    extras_require={
+    install_requires = ['dukpy'],
+    extras_require = {
         'socks': ['PySocks'],
     }
 )
diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ddf12be953..ee30644b30 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -3,6 +3,8 @@
 from ..common import *
 from ..extractor import VideoExtractor
 
+import dukpy
+from urllib.parse import urlparse, parse_qs, urlencode
 from xml.dom.minidom import parseString
 
 class YouTube(VideoExtractor):
@@ -68,45 +70,32 @@ class YouTube(VideoExtractor):
          'audio_encoding': 'AAC', 'audio_bitrate': '24'},
     ]
 
+    def dethrottle(js, url):
+        def n_to_n(js, n):
+            # Examples:
+            #   yma - https://www.youtube.com/s/player/84314bef/player_ias.vflset/en_US/base.js
+            #   Xka - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
+            f1 = match1(js, r'a\.set\("n",b\),[$\w]+\.length\|\|([$\w]+)\(""\)')
+            f1def = match1(js, r'\W%s=(function\(\w+\).+?\)});' % re.escape(f1))
+            n = dukpy.evaljs('(%s)("%s")' % (f1def, n))
+            return n
+
+        u = urlparse(url)
+        qs = parse_qs(u.query)
+        n = n_to_n(js, qs['n'][0])
+        qs['n'] = [n]
+        return u._replace(query=urlencode(qs, doseq=True)).geturl()
+
     def s_to_sig(js, s):
         # Examples:
-        # - https://www.youtube.com/yts/jsbin/player-da_DK-vflWlK-zq/base.js
-        # - https://www.youtube.com/yts/jsbin/player-vflvABTsY/da_DK/base.js
-        # - https://www.youtube.com/yts/jsbin/player-vfls4aurX/da_DK/base.js
-        # - https://www.youtube.com/yts/jsbin/player_ias-vfl_RGK2l/en_US/base.js
-        # - https://www.youtube.com/yts/jsbin/player-vflRjqq_w/da_DK/base.js
-        # - https://www.youtube.com/yts/jsbin/player_ias-vfl-jbnrr/da_DK/base.js
-        # - https://www.youtube.com/s/player/0b643cd1/player_ias.vflset/sv_SE/base.js
-        # - https://www.youtube.com/s/player/50e823fc/player_ias.vflset/sv_SE/base.js
-        # - https://www.youtube.com/s/player/3b5d5649/player_ias.vflset/sv_SE/base.js
-        # - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
-        def tr_js(code):
-            code = re.sub(r'function', r'def', code)
-            # add prefix '_sig_' to prevent namespace pollution
-            code = re.sub(r'(\W)([$\w][$\w][$\w]?)\(', r'\1_sig_\2(', code)
-            code = re.sub(r'\$', '_dollar', code)
-            code = re.sub(r'\{', r': ', code)
-            code = re.sub(r'\}', r'\n', code)
-            code = re.sub(r'var\s+', r'', code)
-            code = re.sub(r'(\w+).join\(""\)', r'"".join(\1)', code)
-            code = re.sub(r'(\w+).length', r'len(\1)', code)
-            code = re.sub(r'(\w+).slice\((\w+)\)', r'\1[\2:]', code)
-            code = re.sub(r'(\w+).splice\((\w+),(\w+)\)', r'del \1[\2:\2+\3]', code)
-            code = re.sub(r'(\w+).split\(""\)', r'list(\1)', code)
-            return code
-
-        js = js.replace('\n', ' ')
-        f1 = match1(js, r'\.set\(\w+\.sp,encodeURIComponent\(([$\w]+)') or \
-            match1(js, r'\.set\(\w+\.sp,\(0,window\.encodeURIComponent\)\(([$\w]+)') or \
-            match1(js, r'\.set\(\w+\.sp,([$\w]+)\(\w+\.s\)\)') or \
-            match1(js, r'"signature",([$\w]+)\(\w+\.\w+\)') or \
-            match1(js, r'=([$\w]+)\(decodeURIComponent\(')
-        f1def = match1(js, r'function %s(\(\w+\)\{[^\{]+\})' % re.escape(f1)) or \
-                match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
-        f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)
+        #   BPa - https://www.youtube.com/s/player/84314bef/player_ias.vflset/en_US/base.js
+        #   Xva - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
+        js_code = ''
+        f1 = match1(js, r'=([$\w]+)\(decodeURIComponent\(')
+        f1def = match1(js, r'\W%s=function(\(\w+\)\{[^\{]+\})' % re.escape(f1))
+        f1def = re.sub(r'([$\w]+\.)([$\w]+\(\w+,\d+\))', r'\2', f1def)  # remove . prefix
         f1def = 'function %s%s' % (f1, f1def)
-        code = tr_js(f1def)
-        f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))
+        f2s = set(re.findall(r'([$\w]+)\(\w+,\d+\)', f1def))  # find all invoked function names
         for f2 in f2s:
             f2e = re.escape(f2)
             f2def = re.search(r'[^$\w]%s:function\((\w+,\w+)\)(\{[^\{\}]+\})' % f2e, js)
@@ -115,13 +104,10 @@ def tr_js(code):
             else:
                 f2def = re.search(r'[^$\w]%s:function\((\w+)\)(\{[^\{\}]+\})' % f2e, js)
                 f2def = 'function {}({},b){}'.format(f2e, f2def.group(1), f2def.group(2))
-            f2 = re.sub(r'\$', '_dollar', f2)  # replace dollar sign
-            code = code + 'global _sig_%s\n' % f2 + tr_js(f2def)
-
-        f1 = re.sub(r'\$', '_dollar', f1)  # replace dollar sign
-        code = code + '_sig=_sig_%s(s)' % f1
-        exec(code, globals(), locals())
-        return locals()['_sig']
+            js_code += f2def + ';'
+        js_code += f1def + ';%s("%s")' % (f1, s)
+        sig = dukpy.evaljs(js_code)
+        return sig
 
     def chunk_by_range(url, size):
         urls = []
@@ -209,6 +195,7 @@ def prepare(self, **kwargs):
             raise
         elif video_info['status'] == ['ok']:
             if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
+                # FIXME: this is basically dead code, use_cipher_signature is always true
                 self.title = parse.unquote_plus(json.loads(video_info["player_response"][0])["videoDetails"]["title"])
                 # Parse video page (for DASH)
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
@@ -259,23 +246,30 @@ def prepare(self, **kwargs):
                         self.html5player = None
 
             else:
-                # Parse video page instead
+                # Extract from video page
+                logging.debug('Extracting from the video page...')
                 video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
 
-                try:  # FIXME: we should extract ytInitialPlayerResponse more reliably
-                    ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
+                try:
+                    jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)
                 except:
-                    ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
+                    log.wtf('[Failed] Unable to find base.js on the video page')
+                # FIXME: do we still need this?
+                jsUrl = jsUrl.replace('\/', '/')  # unescape URL (for age-restricted videos)
+                self.html5player = 'https://www.youtube.com' + jsUrl
+                logging.debug('Retrieving the player code...')
+                self.js = get_content(self.html5player).replace('\n', ' ')
+
+                logging.debug('Loading ytInitialPlayerResponse...')
+                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>)', video_page).group(1))
 
+                # Get the video title
                 self.title = ytInitialPlayerResponse["videoDetails"]["title"]
-                if re.search('([^"]*/base\.js)"', video_page):
-                    self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
-                else:
-                    self.html5player = None
 
                 stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         elif video_info['status'] == ['fail']:
+            # FIXME: this is basically dead code, status is always ok
             logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])
             if video_info['errorcode'] == ['150']:
                 # FIXME: still relevant?
@@ -327,7 +321,7 @@ def prepare(self, **kwargs):
             log.wtf('[Failed] Invalid status.', exit_code=None)
             raise
 
-        # YouTube Live
+        # FIXME: YouTube Live
         if ytplayer_config and (ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1'):
             if 'hlsvp' in ytplayer_config['args']:
                 hlsvp = ytplayer_config['args']['hlsvp']
@@ -343,6 +337,7 @@ def prepare(self, **kwargs):
 
         for stream in stream_list:
             if isinstance(stream, str):
+                # FIXME: dead code?
                 metadata = parse.parse_qs(stream)
                 stream_itag = metadata['itag'][0]
                 self.streams[stream_itag] = {
@@ -357,22 +352,29 @@ def prepare(self, **kwargs):
                     'container': mime_to_container(metadata['type'][0].split(';')[0]),
                 }
             else:
-                stream_itag = str(stream['itag'])
-                self.streams[stream_itag] = {
+                if 'signatureCipher' in stream:
+                    logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                    qs = parse_qs(stream['signatureCipher'])
+                    #logging.debug(qs)
+                    sp = qs['sp'][0]
+                    sig = self.__class__.s_to_sig(self.js, qs['s'][0])
+                    url = qs['url'][0] + '&{}={}'.format(sp, sig)
+                elif 'url' in stream:
+                    url = stream['url']
+                else:
+                    log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
+                url = self.__class__.dethrottle(self.js, url)
+
+                self.streams[str(stream['itag'])] = {
                     'itag': str(stream['itag']),
-                    'url': stream['url'] if 'url' in stream else None,
-                    'sig': None,
-                    's': None,
+                    'url': url,
                     'quality': stream['quality'],
                     'type': stream['mimeType'],
                     'mime': stream['mimeType'].split(';')[0],
                     'container': mime_to_container(stream['mimeType'].split(';')[0]),
                 }
-                if 'signatureCipher' in stream:
-                    self.streams[stream_itag].update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
-                                                           for _ in stream['signatureCipher'].split('&')]))
 
-        # Prepare caption tracks
+        # FIXME: Prepare caption tracks
         try:
             try:
                 caption_tracks = json.loads(ytplayer_config['args']['player_response'])['captions']['playerCaptionsTracklistRenderer']['captionTracks']
@@ -408,6 +410,7 @@ def prepare(self, **kwargs):
 
         # Prepare DASH streams (NOTE: not every video has DASH streams!)
         try:
+            # FIXME: dead code?
             dashmpd = ytplayer_config['args']['dashmpd']
             dash_xml = parseString(get_content(dashmpd))
             for aset in dash_xml.getElementsByTagName('AdaptationSet'):
@@ -473,12 +476,8 @@ def prepare(self, **kwargs):
                             'size': int(dash_size) + int(dash_webm_a_size)
                         }
         except:
-            # VEVO
-            if not self.html5player: return
-            self.html5player = self.html5player.replace('\/', '/') # unescape URL (for age-restricted videos)
-            self.js = get_content(self.html5player)
-
             try:
+                # FIXME: dead code?
                 # Video info from video page (not always available)
                 streams = [dict([(i.split('=')[0],
                                   parse.unquote(i.split('=')[1]))
@@ -486,6 +485,7 @@ def prepare(self, **kwargs):
                            for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
             except:
                 if 'adaptive_fmts' in video_info:
+                    # FIXME: dead code?
                     streams = [dict([(i.split('=')[0],
                                       parse.unquote(i.split('=')[1]))
                                      for i in afmt.split('&')])
@@ -493,12 +493,15 @@ def prepare(self, **kwargs):
                 else:
                     try:
                         try:
+                            # FIXME: dead code?
                             streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
                         except:
                             streams = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
                     except:  # no DASH stream at all
+                        # FIXME: dead code?
                         return
 
+                    # FIXME: dead code?
                     # streams without contentLength got broken urls, just remove them (#2767)
                     streams = [stream for stream in streams if 'contentLength' in stream]
 
@@ -523,34 +526,33 @@ def prepare(self, **kwargs):
                         del stream['contentLength']
                         del stream['initRange']
                         del stream['indexRange']
+
                         if 'signatureCipher' in stream:
-                            stream.update(dict([(_.split('=')[0], parse.unquote(_.split('=')[1]))
-                                                for _ in stream['signatureCipher'].split('&')]))
-                            del stream['signatureCipher']
+                            logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                            qs = parse_qs(stream['signatureCipher'])
+                            #logging.debug(qs)
+                            sp = qs['sp'][0]
+                            sig = self.__class__.s_to_sig(self.js, qs['s'][0])
+                            url = qs['url'][0] + '&ratebypass=yes&{}={}'.format(sp, sig)
+                        elif 'url' in stream:
+                            url = stream['url']
+                        else:
+                            log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
+                        url = self.__class__.dethrottle(self.js, url)
+                        stream['url'] = url
 
-            for stream in streams: # get over speed limiting
-                stream['url'] += '&ratebypass=yes'
             for stream in streams: # audio
                 if stream['type'].startswith('audio/mp4'):
                     dash_mp4_a_url = stream['url']
-                    if 's' in stream:
-                        sig = self.__class__.s_to_sig(self.js, stream['s'])
-                        dash_mp4_a_url += '&sig={}'.format(sig)
                     dash_mp4_a_size = stream['clen']
                 elif stream['type'].startswith('audio/webm'):
                     dash_webm_a_url = stream['url']
-                    if 's' in stream:
-                        sig = self.__class__.s_to_sig(self.js, stream['s'])
-                        dash_webm_a_url += '&sig={}'.format(sig)
                     dash_webm_a_size = stream['clen']
             for stream in streams: # video
                 if 'size' in stream:
                     if stream['type'].startswith('video/mp4'):
                         mimeType = 'video/mp4'
                         dash_url = stream['url']
-                        if 's' in stream:
-                            sig = self.__class__.s_to_sig(self.js, stream['s'])
-                            dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
                         dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
@@ -567,9 +569,6 @@ def prepare(self, **kwargs):
                     elif stream['type'].startswith('video/webm'):
                         mimeType = 'video/webm'
                         dash_url = stream['url']
-                        if 's' in stream:
-                            sig = self.__class__.s_to_sig(self.js, stream['s'])
-                            dash_url += '&sig={}'.format(sig)
                         dash_size = stream['clen']
                         itag = stream['itag']
                         audio_url = None
@@ -610,15 +609,6 @@ def extract(self, **kwargs):
 
         if stream_id in self.streams:
             src = self.streams[stream_id]['url']
-            if self.streams[stream_id]['sig'] is not None:
-                sig = self.streams[stream_id]['sig']
-                src += '&sig={}'.format(sig)
-            elif self.streams[stream_id]['s'] is not None:
-                if not hasattr(self, 'js'):
-                    self.js = get_content(self.html5player)
-                s = self.streams[stream_id]['s']
-                sig = self.__class__.s_to_sig(self.js, s)
-                src += '&sig={}'.format(sig)
 
             self.streams[stream_id]['src'] = [src]
             self.streams[stream_id]['size'] = urls_size(self.streams[stream_id]['src'])
diff --git a/tests/test.py b/tests/test.py
index e8a378f528..8d348fbc00 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -36,9 +36,9 @@ def test_youtube(self):
         #    'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
         #    info_only=True
         #)
-        #youtube.download(
-        #    'https://www.youtube.com/watch?v=Fpr4fQSh1cc', info_only=True
-        #)
+        youtube.download(
+            'https://www.youtube.com/watch?v=oRdxUFDoQe0', info_only=True
+        )
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac44560432', info_only=True)

From b0e6f0cadcfa0598ea5954334b87dab5c76ae238 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 14:43:43 +0200
Subject: [PATCH 1184/1225] [youtube] remove dead code

---
 src/you_get/extractors/youtube.py | 397 ++++++------------------------
 1 file changed, 79 insertions(+), 318 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ee30644b30..f5aaf50ec5 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -182,204 +182,54 @@ def prepare(self, **kwargs):
         if re.search('\Wlist=', self.url) and not kwargs.get('playlist'):
             log.w('This video is from a playlist. (use --playlist to download all videos in the playlist.)')
 
-        # Get video info
-        # 'eurl' is a magic parameter that can bypass age restriction
-        # full form: 'eurl=https%3A%2F%2Fyoutube.googleapis.com%2Fv%2F{VIDEO_ID}'
-        #video_info = parse.parse_qs(get_content('https://www.youtube.com/get_video_info?video_id={}&eurl=https%3A%2F%2Fy'.format(self.vid)))
-        #logging.debug('STATUS: %s' % video_info['status'][0])
-        video_info = {'status': ['ok'], 'use_cipher_signature': 'True'}
-
-        ytplayer_config = None
-        if 'status' not in video_info:
-            log.wtf('[Failed] Unknown status.', exit_code=None)
-            raise
-        elif video_info['status'] == ['ok']:
-            if 'use_cipher_signature' not in video_info or video_info['use_cipher_signature'] == ['False']:
-                # FIXME: this is basically dead code, use_cipher_signature is always true
-                self.title = parse.unquote_plus(json.loads(video_info["player_response"][0])["videoDetails"]["title"])
-                # Parse video page (for DASH)
-                video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-                try:
-                    try:
-                        # Complete ytplayer_config
-                        ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+?});', video_page).group(1))
-
-                        # Workaround: get_video_info returns bad s. Why?
-                        if 'url_encoded_fmt_stream_map' not in ytplayer_config['args']:
-                            stream_list = json.loads(ytplayer_config['args']['player_response'])['streamingData']['formats']
-                        else:
-                            stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
-                        #stream_list = ytplayer_config['args']['adaptive_fmts'].split(',')
-
-                        if 'assets' in ytplayer_config:
-                            self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
-                        elif re.search('([^"]*/base\.js)"', video_page):
-                            self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
-                            self.html5player = self.html5player.replace('\/', '/') # unescape URL
-                        else:
-                            self.html5player = None
-
-                    except:
-                        # ytplayer_config = {args:{raw_player_response:ytInitialPlayerResponse}}
-                        try:  # FIXME: we should extract ytInitialPlayerResponse more reliably
-                            ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});</script>', video_page).group(1))
-                        except:
-                            ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});', video_page).group(1))
-
-                        stream_list = ytInitialPlayerResponse['streamingData']['formats']
-                        #stream_list = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
-
-                        if re.search('([^"]*/base\.js)"', video_page):
-                            self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
-                        else:
-                            self.html5player = None
-
-                except:
-                    if 'url_encoded_fmt_stream_map' not in video_info:
-                        stream_list = json.loads(video_info['player_response'][0])['streamingData']['formats']
-                    else:
-                        stream_list = video_info['url_encoded_fmt_stream_map'][0].split(',')
-
-                    if re.search('([^"]*/base\.js)"', video_page):
-                        self.html5player = 'https://www.youtube.com' + re.search('([^"]*/base\.js)"', video_page).group(1)
-                    else:
-                        self.html5player = None
+        # Extract from video page
+        logging.debug('Extracting from the video page...')
+        video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
 
-            else:
-                # Extract from video page
-                logging.debug('Extracting from the video page...')
-                video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-
-                try:
-                    jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)
-                except:
-                    log.wtf('[Failed] Unable to find base.js on the video page')
-                # FIXME: do we still need this?
-                jsUrl = jsUrl.replace('\/', '/')  # unescape URL (for age-restricted videos)
-                self.html5player = 'https://www.youtube.com' + jsUrl
-                logging.debug('Retrieving the player code...')
-                self.js = get_content(self.html5player).replace('\n', ' ')
-
-                logging.debug('Loading ytInitialPlayerResponse...')
-                ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>)', video_page).group(1))
-
-                # Get the video title
-                self.title = ytInitialPlayerResponse["videoDetails"]["title"]
-
-                stream_list = ytInitialPlayerResponse['streamingData']['formats']
-
-        elif video_info['status'] == ['fail']:
-            # FIXME: this is basically dead code, status is always ok
-            logging.debug('ERRORCODE: %s' % video_info['errorcode'][0])
-            if video_info['errorcode'] == ['150']:
-                # FIXME: still relevant?
-                if cookies:
-                    # Load necessary cookies into headers (for age-restricted videos)
-                    consent, ssid, hsid, sid = 'YES', '', '', ''
-                    for cookie in cookies:
-                        if cookie.domain.endswith('.youtube.com'):
-                            if cookie.name == 'SSID':
-                                ssid = cookie.value
-                            elif cookie.name == 'HSID':
-                                hsid = cookie.value
-                            elif cookie.name == 'SID':
-                                sid = cookie.value
-                    cookie_str = 'CONSENT=%s; SSID=%s; HSID=%s; SID=%s' % (consent, ssid, hsid, sid)
-
-                    video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid,
-                                             headers={'Cookie': cookie_str})
-                else:
-                    video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
-
-                try:
-                    ytplayer_config = json.loads(re.search('ytplayer.config\s*=\s*([^\n]+});ytplayer', video_page).group(1))
-                except:
-                    msg = re.search('class="message">([^<]+)<', video_page).group(1)
-                    log.wtf('[Failed] Got message "%s". Try to login with --cookies.' % msg.strip())
-
-                if 'title' in ytplayer_config['args']:
-                    # 150 Restricted from playback on certain sites
-                    # Parse video page instead
-                    self.title = ytplayer_config['args']['title']
-                    self.html5player = 'https://www.youtube.com' + ytplayer_config['assets']['js']
-                    stream_list = ytplayer_config['args']['url_encoded_fmt_stream_map'].split(',')
-                else:
-                    log.wtf('[Error] The uploader has not made this video available in your country.', exit_code=None)
-                    raise
-                    #self.title = re.search('<meta name="title" content="([^"]+)"', video_page).group(1)
-                    #stream_list = []
+        try:
+            jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)
+        except:
+            log.wtf('[Failed] Unable to find base.js on the video page')
+        # FIXME: do we still need this?
+        jsUrl = jsUrl.replace('\/', '/')  # unescape URL (for age-restricted videos)
+        self.html5player = 'https://www.youtube.com' + jsUrl
+        logging.debug('Retrieving the player code...')
+        self.js = get_content(self.html5player).replace('\n', ' ')
 
-            elif video_info['errorcode'] == ['100']:
-                log.wtf('[Failed] This video does not exist.', exit_code=None) #int(video_info['errorcode'][0])
-                raise
+        logging.debug('Loading ytInitialPlayerResponse...')
+        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>)', video_page).group(1))
 
-            else:
-                log.wtf('[Failed] %s' % video_info['reason'][0], exit_code=None) #int(video_info['errorcode'][0])
-                raise
+        # Get the video title
+        self.title = ytInitialPlayerResponse["videoDetails"]["title"]
 
-        else:
-            log.wtf('[Failed] Invalid status.', exit_code=None)
-            raise
-
-        # FIXME: YouTube Live
-        if ytplayer_config and (ytplayer_config['args'].get('livestream') == '1' or ytplayer_config['args'].get('live_playback') == '1'):
-            if 'hlsvp' in ytplayer_config['args']:
-                hlsvp = ytplayer_config['args']['hlsvp']
-            else:
-                player_response= json.loads(ytplayer_config['args']['player_response'])
-                log.e('[Failed] %s' % player_response['playabilityStatus']['reason'], exit_code=1)
-
-            if 'info_only' in kwargs and kwargs['info_only']:
-                return
-            else:
-                download_url_ffmpeg(hlsvp, self.title, 'mp4')
-                exit(0)
+        stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         for stream in stream_list:
-            if isinstance(stream, str):
-                # FIXME: dead code?
-                metadata = parse.parse_qs(stream)
-                stream_itag = metadata['itag'][0]
-                self.streams[stream_itag] = {
-                    'itag': metadata['itag'][0],
-                    'url': metadata['url'][0],
-                    'sig': metadata['sig'][0] if 'sig' in metadata else None,
-                    's': metadata['s'][0] if 's' in metadata else None,
-                    'quality': metadata['quality'][0] if 'quality' in metadata else None,
-                    #'quality': metadata['quality_label'][0] if 'quality_label' in metadata else None,
-                    'type': metadata['type'][0],
-                    'mime': metadata['type'][0].split(';')[0],
-                    'container': mime_to_container(metadata['type'][0].split(';')[0]),
-                }
+            if 'signatureCipher' in stream:
+                logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                qs = parse_qs(stream['signatureCipher'])
+                #logging.debug(qs)
+                sp = qs['sp'][0]
+                sig = self.__class__.s_to_sig(self.js, qs['s'][0])
+                url = qs['url'][0] + '&{}={}'.format(sp, sig)
+            elif 'url' in stream:
+                url = stream['url']
             else:
-                if 'signatureCipher' in stream:
-                    logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
-                    qs = parse_qs(stream['signatureCipher'])
-                    #logging.debug(qs)
-                    sp = qs['sp'][0]
-                    sig = self.__class__.s_to_sig(self.js, qs['s'][0])
-                    url = qs['url'][0] + '&{}={}'.format(sp, sig)
-                elif 'url' in stream:
-                    url = stream['url']
-                else:
-                    log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
-                url = self.__class__.dethrottle(self.js, url)
-
-                self.streams[str(stream['itag'])] = {
-                    'itag': str(stream['itag']),
-                    'url': url,
-                    'quality': stream['quality'],
-                    'type': stream['mimeType'],
-                    'mime': stream['mimeType'].split(';')[0],
-                    'container': mime_to_container(stream['mimeType'].split(';')[0]),
-                }
+                log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
+            url = self.__class__.dethrottle(self.js, url)
+
+            self.streams[str(stream['itag'])] = {
+                'itag': str(stream['itag']),
+                'url': url,
+                'quality': stream['quality'],
+                'type': stream['mimeType'],
+                'mime': stream['mimeType'].split(';')[0],
+                'container': mime_to_container(stream['mimeType'].split(';')[0]),
+            }
 
         # FIXME: Prepare caption tracks
         try:
-            try:
-                caption_tracks = json.loads(ytplayer_config['args']['player_response'])['captions']['playerCaptionsTracklistRenderer']['captionTracks']
-            except:
-                caption_tracks = ytInitialPlayerResponse['captions']['playerCaptionsTracklistRenderer']['captionTracks']
+            caption_tracks = ytInitialPlayerResponse['captions']['playerCaptionsTracklistRenderer']['captionTracks']
             for ct in caption_tracks:
                 ttsurl, lang = ct['baseUrl'], ct['languageCode']
 
@@ -408,138 +258,49 @@ def prepare(self, **kwargs):
                 self.caption_tracks[lang] = srt
         except: pass
 
-        # Prepare DASH streams (NOTE: not every video has DASH streams!)
-        try:
+        # Prepare DASH streams
+        if 'adaptiveFormats' in ytInitialPlayerResponse['streamingData']:
+            streams = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
+
             # FIXME: dead code?
-            dashmpd = ytplayer_config['args']['dashmpd']
-            dash_xml = parseString(get_content(dashmpd))
-            for aset in dash_xml.getElementsByTagName('AdaptationSet'):
-                mimeType = aset.getAttribute('mimeType')
-                if mimeType == 'audio/mp4':
-                    rep = aset.getElementsByTagName('Representation')[-1]
-                    burls = rep.getElementsByTagName('BaseURL')
-                    dash_mp4_a_url = burls[0].firstChild.nodeValue
-                    dash_mp4_a_size = burls[0].getAttribute('yt:contentLength')
-                    if not dash_mp4_a_size:
-                        try: dash_mp4_a_size = url_size(dash_mp4_a_url)
-                        except: continue
-                elif mimeType == 'audio/webm':
-                    rep = aset.getElementsByTagName('Representation')[-1]
-                    burls = rep.getElementsByTagName('BaseURL')
-                    dash_webm_a_url = burls[0].firstChild.nodeValue
-                    dash_webm_a_size = burls[0].getAttribute('yt:contentLength')
-                    if not dash_webm_a_size:
-                        try: dash_webm_a_size = url_size(dash_webm_a_url)
-                        except: continue
-                elif mimeType == 'video/mp4':
-                    for rep in aset.getElementsByTagName('Representation'):
-                        w = int(rep.getAttribute('width'))
-                        h = int(rep.getAttribute('height'))
-                        itag = rep.getAttribute('id')
-                        burls = rep.getElementsByTagName('BaseURL')
-                        dash_url = burls[0].firstChild.nodeValue
-                        dash_size = burls[0].getAttribute('yt:contentLength')
-                        if not dash_size:
-                            try: dash_size = url_size(dash_url)
-                            except: continue
-                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
-                        dash_mp4_a_urls = self.__class__.chunk_by_range(dash_mp4_a_url, int(dash_mp4_a_size))
-                        self.dash_streams[itag] = {
-                            'quality': '%sx%s' % (w, h),
-                            'itag': itag,
-                            'type': mimeType,
-                            'mime': mimeType,
-                            'container': 'mp4',
-                            'src': [dash_urls, dash_mp4_a_urls],
-                            'size': int(dash_size) + int(dash_mp4_a_size)
-                        }
-                elif mimeType == 'video/webm':
-                    for rep in aset.getElementsByTagName('Representation'):
-                        w = int(rep.getAttribute('width'))
-                        h = int(rep.getAttribute('height'))
-                        itag = rep.getAttribute('id')
-                        burls = rep.getElementsByTagName('BaseURL')
-                        dash_url = burls[0].firstChild.nodeValue
-                        dash_size = burls[0].getAttribute('yt:contentLength')
-                        if not dash_size:
-                            try: dash_size = url_size(dash_url)
-                            except: continue
-                        dash_urls = self.__class__.chunk_by_range(dash_url, int(dash_size))
-                        dash_webm_a_urls = self.__class__.chunk_by_range(dash_webm_a_url, int(dash_webm_a_size))
-                        self.dash_streams[itag] = {
-                            'quality': '%sx%s' % (w, h),
-                            'itag': itag,
-                            'type': mimeType,
-                            'mime': mimeType,
-                            'container': 'webm',
-                            'src': [dash_urls, dash_webm_a_urls],
-                            'size': int(dash_size) + int(dash_webm_a_size)
-                        }
-        except:
-            try:
-                # FIXME: dead code?
-                # Video info from video page (not always available)
-                streams = [dict([(i.split('=')[0],
-                                  parse.unquote(i.split('=')[1]))
-                                 for i in afmt.split('&')])
-                           for afmt in ytplayer_config['args']['adaptive_fmts'].split(',')]
-            except:
-                if 'adaptive_fmts' in video_info:
-                    # FIXME: dead code?
-                    streams = [dict([(i.split('=')[0],
-                                      parse.unquote(i.split('=')[1]))
-                                     for i in afmt.split('&')])
-                               for afmt in video_info['adaptive_fmts'][0].split(',')]
+            # streams without contentLength got broken urls, just remove them (#2767)
+            streams = [stream for stream in streams if 'contentLength' in stream]
+
+            for stream in streams:
+                stream['itag'] = str(stream['itag'])
+                if 'qualityLabel' in stream:
+                    stream['quality_label'] = stream['qualityLabel']
+                    del stream['qualityLabel']
+                if 'width' in stream:
+                    stream['size'] = '{}x{}'.format(stream['width'], stream['height'])
+                    del stream['width']
+                    del stream['height']
+                stream['type'] = stream['mimeType']
+                stream['clen'] = stream['contentLength']
+                stream['init'] = '{}-{}'.format(
+                    stream['initRange']['start'],
+                    stream['initRange']['end'])
+                stream['index'] = '{}-{}'.format(
+                    stream['indexRange']['start'],
+                    stream['indexRange']['end'])
+                del stream['mimeType']
+                del stream['contentLength']
+                del stream['initRange']
+                del stream['indexRange']
+
+                if 'signatureCipher' in stream:
+                    logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                    qs = parse_qs(stream['signatureCipher'])
+                    #logging.debug(qs)
+                    sp = qs['sp'][0]
+                    sig = self.__class__.s_to_sig(self.js, qs['s'][0])
+                    url = qs['url'][0] + '&ratebypass=yes&{}={}'.format(sp, sig)
+                elif 'url' in stream:
+                    url = stream['url']
                 else:
-                    try:
-                        try:
-                            # FIXME: dead code?
-                            streams = json.loads(video_info['player_response'][0])['streamingData']['adaptiveFormats']
-                        except:
-                            streams = ytInitialPlayerResponse['streamingData']['adaptiveFormats']
-                    except:  # no DASH stream at all
-                        # FIXME: dead code?
-                        return
-
-                    # FIXME: dead code?
-                    # streams without contentLength got broken urls, just remove them (#2767)
-                    streams = [stream for stream in streams if 'contentLength' in stream]
-
-                    for stream in streams:
-                        stream['itag'] = str(stream['itag'])
-                        if 'qualityLabel' in stream:
-                            stream['quality_label'] = stream['qualityLabel']
-                            del stream['qualityLabel']
-                        if 'width' in stream:
-                            stream['size'] = '{}x{}'.format(stream['width'], stream['height'])
-                            del stream['width']
-                            del stream['height']
-                        stream['type'] = stream['mimeType']
-                        stream['clen'] = stream['contentLength']
-                        stream['init'] = '{}-{}'.format(
-                            stream['initRange']['start'],
-                            stream['initRange']['end'])
-                        stream['index'] = '{}-{}'.format(
-                            stream['indexRange']['start'],
-                            stream['indexRange']['end'])
-                        del stream['mimeType']
-                        del stream['contentLength']
-                        del stream['initRange']
-                        del stream['indexRange']
-
-                        if 'signatureCipher' in stream:
-                            logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
-                            qs = parse_qs(stream['signatureCipher'])
-                            #logging.debug(qs)
-                            sp = qs['sp'][0]
-                            sig = self.__class__.s_to_sig(self.js, qs['s'][0])
-                            url = qs['url'][0] + '&ratebypass=yes&{}={}'.format(sp, sig)
-                        elif 'url' in stream:
-                            url = stream['url']
-                        else:
-                            log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
-                        url = self.__class__.dethrottle(self.js, url)
-                        stream['url'] = url
+                    log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
+                url = self.__class__.dethrottle(self.js, url)
+                stream['url'] = url
 
             for stream in streams: # audio
                 if stream['type'].startswith('audio/mp4'):

From 2aaf38ce239418fa40a72cf6b3a84ad66c9a4cba Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 17:03:31 +0200
Subject: [PATCH 1185/1225] [youtube] check playabilityStatus and ask for
 cookies if needed

---
 src/you_get/extractors/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index f5aaf50ec5..1d370ec2e4 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -190,8 +190,6 @@ def prepare(self, **kwargs):
             jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)
         except:
             log.wtf('[Failed] Unable to find base.js on the video page')
-        # FIXME: do we still need this?
-        jsUrl = jsUrl.replace('\/', '/')  # unescape URL (for age-restricted videos)
         self.html5player = 'https://www.youtube.com' + jsUrl
         logging.debug('Retrieving the player code...')
         self.js = get_content(self.html5player).replace('\n', ' ')
@@ -202,6 +200,14 @@ def prepare(self, **kwargs):
         # Get the video title
         self.title = ytInitialPlayerResponse["videoDetails"]["title"]
 
+        # Check the status
+        playabilityStatus = ytInitialPlayerResponse['playabilityStatus']
+        status = playabilityStatus['status']
+        logging.debug('status: %s' % status)
+        if status != 'OK':
+            # If cookies are loaded, status should be OK
+            log.wtf('[Failed] %s (use --cookies to load cookies)' % playabilityStatus['reason'])
+
         stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         for stream in stream_list:

From 2c976407e5e84becd2b3cc85eccbd3de04ac7f56 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 17:42:54 +0200
Subject: [PATCH 1186/1225] [youtube] show subreason if possible

---
 src/you_get/extractors/youtube.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 1d370ec2e4..a76e0d2e08 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -206,7 +206,13 @@ def prepare(self, **kwargs):
         logging.debug('status: %s' % status)
         if status != 'OK':
             # If cookies are loaded, status should be OK
-            log.wtf('[Failed] %s (use --cookies to load cookies)' % playabilityStatus['reason'])
+            try:
+                subreason = playabilityStatus['errorScreen']['playerErrorMessageRenderer']['subreason']['runs'][0]['text']
+                log.e('[Error] %s (%s)' % (playabilityStatus['reason'], subreason))
+            except:
+                log.e('[Error] %s' % playabilityStatus['reason'])
+            log.e('View the video from a browser and export the cookies, then use --cookies to load cookies.')
+            exit(1)
 
         stream_list = ytInitialPlayerResponse['streamingData']['formats']
 

From 68965e0a962b050d901de10605af8238074c3101 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 18:01:53 +0200
Subject: [PATCH 1187/1225] [youtube] ask for cookies only if status is
 LOGIN_REQUIRED

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index a76e0d2e08..82d271f07d 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -211,7 +211,8 @@ def prepare(self, **kwargs):
                 log.e('[Error] %s (%s)' % (playabilityStatus['reason'], subreason))
             except:
                 log.e('[Error] %s' % playabilityStatus['reason'])
-            log.e('View the video from a browser and export the cookies, then use --cookies to load cookies.')
+            if status == 'LOGIN_REQUIRED':
+                log.e('View the video from a browser and export the cookies, then use --cookies to load cookies.')
             exit(1)
 
         stream_list = ytInitialPlayerResponse['streamingData']['formats']

From 2569aa03f66abb48e348d89a37d7fbeb31adc0c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 18:44:47 +0200
Subject: [PATCH 1188/1225] [youtube] prompt for installing dukpy if
 ImportError

---
 src/you_get/extractors/youtube.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 82d271f07d..38da668b51 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -3,7 +3,12 @@
 from ..common import *
 from ..extractor import VideoExtractor
 
-import dukpy
+try:
+    import dukpy
+except ImportError:
+    log.e('Please install dukpy in order to extract videos from YouTube:')
+    log.e('$ pip install dukpy')
+    exit(0)
 from urllib.parse import urlparse, parse_qs, urlencode
 from xml.dom.minidom import parseString
 

From 4b9c9510849c48984b165a348aea52dc8176a52e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 20:30:31 +0200
Subject: [PATCH 1189/1225] switch from "setup.py test" to "python -m unittest"
 (pytest has a weird 'I/O operation on closed file' bug though)

---
 .github/workflows/python-package.yml | 2 +-
 Makefile                             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 1d9afe9fa5..2ee74663d8 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -26,7 +26,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools
-        pip install flake8 pytest
+        pip install flake8
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |
diff --git a/Makefile b/Makefile
index fe4a238c14..7b433237c6 100644
--- a/Makefile
+++ b/Makefile
@@ -8,7 +8,7 @@ i:
 	@(cd src/; python3 -i -c 'import you_get; print("You-Get %s\n>>> import you_get" % you_get.version.__version__)')
 
 test:
-	$(SETUP) test
+	(cd src; python -m unittest discover -s ../tests)
 
 clean:
 	zenity --question

From 31365ac3686edbfa93cb63f019ffa74cdd64bc7e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 20:56:10 +0200
Subject: [PATCH 1190/1225] add requirements.txt

---
 .gitignore       | 3 ++-
 requirements.txt | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)
 create mode 100644 requirements.txt

diff --git a/.gitignore b/.gitignore
index 716c13b304..57f9412b7c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -79,6 +79,7 @@ _*
 *.ts
 *.webm
 *.xml
+*.json
 /.env
 /.idea
 *.m4a
@@ -88,5 +89,5 @@ _*
 
 *.zip
 
+.emacs*
 .vscode
-
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000..7af04e4691
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,2 @@
+# runtime dependencies
+dukpy

From 450ce6e521c37a21e82c71f8077c8ccfdbc0b105 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 21:04:52 +0200
Subject: [PATCH 1191/1225] version 0.4.1710

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 14ed6a9977..cbb80d9c7b 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1700'
+__version__ = '0.4.1710'

From 08ff14c45547b6b24e50e9643bab7b25b80b492c Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 23 Jun 2024 22:30:16 +0200
Subject: [PATCH 1192/1225] Makefile: stop using python setup.py commands

---
 Makefile | 32 ++++++++------------------------
 1 file changed, 8 insertions(+), 24 deletions(-)

diff --git a/Makefile b/Makefile
index 7b433237c6..a622202414 100644
--- a/Makefile
+++ b/Makefile
@@ -1,11 +1,9 @@
-SETUP = python3 setup.py
-
-.PHONY: default i test clean all html rst build sdist bdist bdist_egg bdist_wheel install release
+.PHONY: default i test clean all html rst build install release
 
 default: i
 
 i:
-	@(cd src/; python3 -i -c 'import you_get; print("You-Get %s\n>>> import you_get" % you_get.version.__version__)')
+	@(cd src; python -i -c 'import you_get; print("You-Get %s\n>>> import you_get" % you_get.version.__version__)')
 
 test:
 	(cd src; python -m unittest discover -s ../tests)
@@ -16,7 +14,7 @@ clean:
 	find . | grep __pycache__ | xargs rm -fr
 	find . | grep .pyc | xargs rm -f
 
-all: build sdist bdist bdist_egg bdist_wheel
+all: build
 
 html:
 	pandoc README.md > README.html
@@ -25,25 +23,11 @@ rst:
 	pandoc -s -t rst README.md > README.rst
 
 build:
-	$(SETUP) build
-
-sdist:
-	$(SETUP) sdist
-
-bdist:
-	$(SETUP) bdist
-
-bdist_egg:
-	$(SETUP) bdist_egg
-
-bdist_wheel:
-	$(SETUP) bdist_wheel
+	python -m build
 
 install:
-	$(SETUP) install --user --prefix=
+	python -m pip install .
 
-release:
-	#zenity --question
-	$(SETUP) sdist bdist_wheel
-	echo 'Upload new version to PyPI using:'
-	echo '	twine upload --sign dist/you-get-VERSION.tar.gz dist/you_get-VERSION-py3-none-any.whl'
+release: build
+	@echo 'Upload new version to PyPI using:'
+	@echo '	twine upload --sign dist/you_get-VERSION*'

From ef6a97301328d1001922d865a9bd830c5da85291 Mon Sep 17 00:00:00 2001
From: Rui Chen <rui@chenrui.dev>
Date: Sun, 23 Jun 2024 22:57:59 -0400
Subject: [PATCH 1193/1225] chore: update `MANIFEST.in` to include `contrib`

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 521b023bd0..d316436404 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,3 +4,4 @@ include Makefile
 include README.md
 include you-get
 include you-get.json
+recursive-include contrib *

From edf29ae1ef7906b606b3ae4099582df655c12676 Mon Sep 17 00:00:00 2001
From: Rui Chen <rui@chenrui.dev>
Date: Sun, 23 Jun 2024 22:59:18 -0400
Subject: [PATCH 1194/1225] chore: also include `you-get.plugin.zsh` and
 `CONTRIBUTING.md`

---
 MANIFEST.in | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index d316436404..ed688fde16 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,7 +1,9 @@
 include *.rst
 include *.txt
 include Makefile
+include CONTRIBUTING.md
 include README.md
 include you-get
 include you-get.json
+include you-get.plugin.zsh
 recursive-include contrib *

From ed3f48810db3eb8664f9851effe5f9af241e25a1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 25 Jun 2024 03:33:21 +0200
Subject: [PATCH 1195/1225] [common] update UA

---
 src/you_get/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 752c63f2ae..2e2fbebd89 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -145,7 +145,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/123.0.2420.97'  # Latest Edge
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.2592.68'  # Latest Edge
 }
 
 if sys.stdout.isatty():

From 0c216b3eb40c610784c53561896451775e6a92b1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 25 Jun 2024 05:53:30 +0200
Subject: [PATCH 1196/1225] [youtube] more detailed logging of stream formats

---
 src/you_get/extractors/youtube.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 38da668b51..cca65f0884 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -223,8 +223,9 @@ def prepare(self, **kwargs):
         stream_list = ytInitialPlayerResponse['streamingData']['formats']
 
         for stream in stream_list:
+            logging.debug('Found format: itag=%s' % stream['itag'])
             if 'signatureCipher' in stream:
-                logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                logging.debug('  Parsing signatureCipher for itag=%s...' % stream['itag'])
                 qs = parse_qs(stream['signatureCipher'])
                 #logging.debug(qs)
                 sp = qs['sp'][0]
@@ -233,7 +234,7 @@ def prepare(self, **kwargs):
             elif 'url' in stream:
                 url = stream['url']
             else:
-                log.wtf('No signatureCipher or url for itag=%s' % stream['itag'])
+                log.wtf('  No signatureCipher or url for itag=%s' % stream['itag'])
             url = self.__class__.dethrottle(self.js, url)
 
             self.streams[str(stream['itag'])] = {
@@ -285,15 +286,19 @@ def prepare(self, **kwargs):
             streams = [stream for stream in streams if 'contentLength' in stream]
 
             for stream in streams:
+                logging.debug('Found adaptiveFormat: itag=%s' % stream['itag'])
                 stream['itag'] = str(stream['itag'])
                 if 'qualityLabel' in stream:
                     stream['quality_label'] = stream['qualityLabel']
                     del stream['qualityLabel']
+                    logging.debug('  quality_label: \t%s' % stream['quality_label'])
                 if 'width' in stream:
                     stream['size'] = '{}x{}'.format(stream['width'], stream['height'])
                     del stream['width']
                     del stream['height']
+                    logging.debug('  size: \t%s' % stream['size'])
                 stream['type'] = stream['mimeType']
+                logging.debug('  type: \t%s' % stream['type'])
                 stream['clen'] = stream['contentLength']
                 stream['init'] = '{}-{}'.format(
                     stream['initRange']['start'],
@@ -307,7 +312,7 @@ def prepare(self, **kwargs):
                 del stream['indexRange']
 
                 if 'signatureCipher' in stream:
-                    logging.debug('Parsing signatureCipher for itag=%s...' % stream['itag'])
+                    logging.debug('  Parsing signatureCipher for itag=%s...' % stream['itag'])
                     qs = parse_qs(stream['signatureCipher'])
                     #logging.debug(qs)
                     sp = qs['sp'][0]

From 882b9c07f80f0b666b3f1c4fac0e7625c5c0399f Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Thu, 27 Jun 2024 05:53:56 +0200
Subject: [PATCH 1197/1225] [youtube] separate files for autogenerated caption
 tracks

---
 src/you_get/extractors/youtube.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index cca65f0884..e36124c859 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -274,7 +274,10 @@ def prepare(self, **kwargs):
                     srt += '%s --> %s\n' % (start, finish)
                     srt += '%s\n\n' % content
 
-                self.caption_tracks[lang] = srt
+                if 'kind' in ct:
+                    self.caption_tracks[ct['vssId']] = srt  # autogenerated
+                else:
+                    self.caption_tracks[lang] = srt
         except: pass
 
         # Prepare DASH streams

From 1a928fac497de1faabe625ea905efc64a1cf7b06 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 14 Jul 2024 08:28:21 +0200
Subject: [PATCH 1198/1225] [youtube] fix extraction

---
 src/you_get/extractors/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index e36124c859..f87ea31b22 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -80,7 +80,8 @@ def n_to_n(js, n):
             # Examples:
             #   yma - https://www.youtube.com/s/player/84314bef/player_ias.vflset/en_US/base.js
             #   Xka - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
-            f1 = match1(js, r'a\.set\("n",b\),[$\w]+\.length\|\|([$\w]+)\(""\)')
+            #   jma - https://www.youtube.com/s/player/8d9f6215/player_ias.vflset/sv_SE/base.js
+            f1 = match1(js, r',[$\w]+\.length\|\|([$\w]+)\(""\)\)}};')
             f1def = match1(js, r'\W%s=(function\(\w+\).+?\)});' % re.escape(f1))
             n = dukpy.evaljs('(%s)("%s")' % (f1def, n))
             return n

From aba2852f84e8b5ea7bdb06f14e95a9ba4e8e0f84 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 14 Jul 2024 08:29:25 +0200
Subject: [PATCH 1199/1225] version 0.4.1718

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index cbb80d9c7b..742f2def02 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1710'
+__version__ = '0.4.1718'

From 899e2b6b2b0f85a627c8241384ed5d28e43a49c1 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 24 Jul 2024 00:26:54 +0200
Subject: [PATCH 1200/1225] [youtube] sloppy fix

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index f87ea31b22..dd06daf8ad 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -201,7 +201,7 @@ def prepare(self, **kwargs):
         self.js = get_content(self.html5player).replace('\n', ' ')
 
         logging.debug('Loading ytInitialPlayerResponse...')
-        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>)', video_page).group(1))
+        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>|var )', video_page).group(1))
 
         # Get the video title
         self.title = ytInitialPlayerResponse["videoDetails"]["title"]

From 4ab02216cdfde52c9828c48644accde08c85e575 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Wanlin=20Wang=20=E7=8E=8B=E4=B8=87=E9=9C=96?=
 <dgideas@outlook.com>
Date: Thu, 25 Jul 2024 17:20:26 +0800
Subject: [PATCH 1201/1225] Add ytInitialPlayerResponse checker to let user
 easily know problem

---
 src/you_get/extractors/youtube.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index dd06daf8ad..2e31e6466a 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -175,6 +175,16 @@ def download_playlist_by_url(self, url, **kwargs):
                 pass
         # FIXME: show DASH stream sizes (by default) for playlist videos
 
+    def check_playability_response(self, ytInitialPlayerResponse):
+        STATUS_OK = "OK"
+        
+        playerResponseStatus = ytInitialPlayerResponse["playabilityStatus"]["status"]
+        if playerResponseStatus != STATUS_OK:
+            reason = ytInitialPlayerResponse["playabilityStatus"].get("reason", "")
+            raise AssertionError(
+                f"Server refused to provide video details. Returned status: {playerResponseStatus}, reason: {reason}."
+            )
+
     def prepare(self, **kwargs):
         assert self.url or self.vid
 
@@ -202,6 +212,7 @@ def prepare(self, **kwargs):
 
         logging.debug('Loading ytInitialPlayerResponse...')
         ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>|var )', video_page).group(1))
+        self.check_playability_response(ytInitialPlayerResponse)
 
         # Get the video title
         self.title = ytInitialPlayerResponse["videoDetails"]["title"]

From 7acebdab9d6ff76ecb98f6cdf753e6f20d333f00 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 30 Jul 2024 00:28:13 +0200
Subject: [PATCH 1202/1225] [instagram] fix extraction

---
 src/you_get/common.py               | 2 +-
 src/you_get/extractors/instagram.py | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 2e2fbebd89..f6956fadc7 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -145,7 +145,7 @@
     'Accept-Charset': 'UTF-8,*;q=0.5',
     'Accept-Encoding': 'gzip,deflate,sdch',
     'Accept-Language': 'en-US,en;q=0.8',
-    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.2592.68'  # Latest Edge
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/126.0.2592.113'  # Latest Edge
 }
 
 if sys.stdout.isatty():
diff --git a/src/you_get/extractors/instagram.py b/src/you_get/extractors/instagram.py
index 8e261fe791..b0fd6f0fa7 100755
--- a/src/you_get/extractors/instagram.py
+++ b/src/you_get/extractors/instagram.py
@@ -5,8 +5,13 @@
 from ..common import *
 
 def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Edg/126.0.2592.87',
+        'sec-fetch-mode': 'navigate'  # important
+    }
+
     url = r1(r'([^?]*)', url)
-    cont = get_content(url, headers=fake_headers)
+    cont = get_content(url, headers=headers)
 
     vid = r1(r'instagram.com/\w+/([^/]+)', url)
     description = r1(r'<meta property="og:title" content="([^"]*)"', cont) or \
@@ -15,6 +20,8 @@ def instagram_download(url, output_dir='.', merge=True, info_only=False, **kwarg
 
     appId = r1(r'"appId":"(\d+)"', cont)
     media_id = r1(r'"media_id":"(\d+)"', cont)
+    logging.debug('appId: %s' % appId)
+    logging.debug('media_id: %s' % media_id)
 
     api_url = 'https://i.instagram.com/api/v1/media/%s/info/' % media_id
     try:

From 1e304893deea04bedaf32fd750b161a5ad75db5f Mon Sep 17 00:00:00 2001
From: Bryan Honof <bryan@floxdev.com>
Date: Wed, 31 Jul 2024 13:45:22 +0200
Subject: [PATCH 1203/1225] Add Flox as an installation option

---
 README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/README.md b/README.md
index 0db87791ed..a3cb7cea84 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,14 @@ You can install `you-get` easily via:
 # pkg install you-get
 ```
 
+### Option 7: Flox (Mac, Linux, and Windows WSL)
+
+You can install `you-get` easily via:
+
+```
+$ flox install you-get
+```
+
 ### Shell completion
 
 Completion definitions for Bash, Fish and Zsh can be found in [`contrib/completion`](https://github.com/soimort/you-get/tree/develop/contrib/completion). Please consult your shell's manual for how to take advantage of them.

From bc0e680ed5cd86f968641639db4b933fe6744b29 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Fri, 2 Aug 2024 20:20:53 +0200
Subject: [PATCH 1204/1225] [tiktok] support short URLs (vt.tiktok.com)

---
 src/you_get/extractors/tiktok.py | 2 ++
 tests/test.py                    | 8 +++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/tiktok.py b/src/you_get/extractors/tiktok.py
index 3cbb59d347..d1d98c417c 100644
--- a/src/you_get/extractors/tiktok.py
+++ b/src/you_get/extractors/tiktok.py
@@ -16,6 +16,8 @@ def tiktok_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     m = re.match('(https?://)?([^/]+)(/.*)', url)
     host = m.group(2)
     if host != 'www.tiktok.com':  # non-canonical URL
+        if host == 'vt.tiktok.com':  # short URL
+            url = get_location(url)
         vid = r1(r'/video/(\d+)', url)
         url = 'https://www.tiktok.com/@/video/%s/' % vid
         host = 'www.tiktok.com'
diff --git a/tests/test.py b/tests/test.py
index 8d348fbc00..f04b1fe236 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -57,12 +57,14 @@ def test_acfun(self):
         #)
 
     def test_tiktok(self):
-        tiktok.download('https://www.tiktok.com/@nmb48_official/video/6850796940293164290', info_only=True)
-        tiktok.download('https://www.tiktok.com/@/video/6850796940293164290', info_only=True)
-        tiktok.download('https://t.tiktok.com/i18n/share/video/6850796940293164290/', info_only=True)
+        tiktok.download('https://www.tiktok.com/@zukky_48/video/7398162058153315605', info_only=True)
+        tiktok.download('https://www.tiktok.com/@/video/7398162058153315605', info_only=True)
+        tiktok.download('https://t.tiktok.com/i18n/share/video/7398162058153315605/', info_only=True)
+        tiktok.download('https://vt.tiktok.com/ZSYKjKt6M/', info_only=True)
 
     def test_twitter(self):
         twitter.download('https://twitter.com/elonmusk/status/1530516552084234244', info_only=True)
+        twitter.download('https://x.com/elonmusk/status/1530516552084234244', info_only=True)
 
     def test_weibo(self):
         miaopai.download('https://video.weibo.com/show?fid=1034:4825403706245135', info_only=True)

From 4cec20ac208b7c1a0c6433d9d9bbfaa6b77e3148 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 4 Aug 2024 19:06:56 +0200
Subject: [PATCH 1205/1225] [youtube] add self.ua

---
 src/you_get/extractors/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 2e31e6466a..ea365a5aaf 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -177,7 +177,7 @@ def download_playlist_by_url(self, url, **kwargs):
 
     def check_playability_response(self, ytInitialPlayerResponse):
         STATUS_OK = "OK"
-        
+
         playerResponseStatus = ytInitialPlayerResponse["playabilityStatus"]["status"]
         if playerResponseStatus != STATUS_OK:
             reason = ytInitialPlayerResponse["playabilityStatus"].get("reason", "")
@@ -186,6 +186,8 @@ def check_playability_response(self, ytInitialPlayerResponse):
             )
 
     def prepare(self, **kwargs):
+        self.ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.2651.86'
+
         assert self.url or self.vid
 
         if not self.vid and self.url:

From afbadf23014b2955ce70b043930d795c68127b7e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 18 Aug 2024 03:36:58 +0200
Subject: [PATCH 1206/1225] [youtube] update self.ua (fix extraction)

---
 src/you_get/extractors/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ea365a5aaf..0a93d396b8 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -186,7 +186,7 @@ def check_playability_response(self, ytInitialPlayerResponse):
             )
 
     def prepare(self, **kwargs):
-        self.ua = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.2651.86'
+        self.ua = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.103 Mobile Safari/537.36'
 
         assert self.url or self.vid
 
@@ -202,7 +202,7 @@ def prepare(self, **kwargs):
 
         # Extract from video page
         logging.debug('Extracting from the video page...')
-        video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid)
+        video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid, headers={'User-Agent': self.ua})
 
         try:
             jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)

From 72b1a7bce13179f4678654d65e9f7cd9917dcaeb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 19 Aug 2024 18:20:21 +0200
Subject: [PATCH 1207/1225] fix "SyntaxWarning: invalid escape sequence" for
 Python 3.12

---
 src/you_get/common.py               |  2 +-
 src/you_get/extractors/acfun.py     |  4 ++--
 src/you_get/extractors/baidu.py     |  2 +-
 src/you_get/extractors/coub.py      |  2 +-
 src/you_get/extractors/douban.py    |  2 +-
 src/you_get/extractors/embed.py     | 24 ++++++++++++------------
 src/you_get/extractors/funshion.py  |  2 +-
 src/you_get/extractors/ku6.py       |  2 +-
 src/you_get/extractors/kugou.py     | 10 +++++-----
 src/you_get/extractors/kuwo.py      |  4 ++--
 src/you_get/extractors/mgtv.py      |  6 +++---
 src/you_get/extractors/qq.py        |  2 +-
 src/you_get/extractors/sohu.py      |  2 +-
 src/you_get/extractors/tudou.py     |  2 +-
 src/you_get/extractors/tumblr.py    |  2 +-
 src/you_get/extractors/twitter.py   |  2 +-
 src/you_get/extractors/universal.py | 18 +++++++++---------
 src/you_get/extractors/vimeo.py     |  2 +-
 src/you_get/extractors/youku.py     | 10 +++++-----
 src/you_get/extractors/youtube.py   |  6 +++---
 20 files changed, 53 insertions(+), 53 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index f6956fadc7..0b307ddee8 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -715,7 +715,7 @@ def url_save(
                         bar.done()
                     if not force and auto_rename:
                         path, ext = os.path.basename(filepath).rsplit('.', 1)
-                        finder = re.compile(' \([1-9]\d*?\)$')
+                        finder = re.compile(r' \([1-9]\d*?\)$')
                         if (finder.search(path) is None):
                             thisfile = path + ' (1).' + ext
                         else:
diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index cd27592789..5775eb5e74 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -32,7 +32,7 @@ def prepare(self, **kwargs):
                 self.title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
             currentVideoInfo = json_data.get('currentVideoInfo')
 
-        elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", self.url):
+        elif re.match(r"https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", self.url):
             html = get_content(self.url, headers=fake_headers)
             tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
             json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
@@ -180,7 +180,7 @@ def getM3u8UrlFromCurrentVideoInfo(currentVideoInfo):
                 title += " - " + [p.get('title') for p in video_list if p.get('id') == vid][0]
             currentVideoInfo = json_data.get('currentVideoInfo')
             m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
-        elif re.match("https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)", url):
+        elif re.match(r'https?://[^\.]*\.*acfun\.[^\.]+/bangumi/aa(\d+)', url):
             html = get_content(url, headers=fake_headers)
             tag_script = match1(html, r'<script>\s*window\.pageInfo([^<]+)</script>')
             json_text = tag_script[tag_script.find('{') : tag_script.find('};') + 1]
diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 521d5e996f..61b0ad246f 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -116,7 +116,7 @@ def baidu_download(url, output_dir='.', stream_type=None, merge=True, info_only=
         id = r1(r'https?://music.baidu.com/album/(\d+)', url)
         baidu_download_album(id, output_dir, merge, info_only)
 
-    elif re.match('https?://music.baidu.com/song/\d+', url):
+    elif re.match(r'https?://music.baidu.com/song/\d+', url):
         id = r1(r'https?://music.baidu.com/song/(\d+)', url)
         baidu_download_song(id, output_dir, merge, info_only)
 
diff --git a/src/you_get/extractors/coub.py b/src/you_get/extractors/coub.py
index a71cbc18e5..b7becb4577 100644
--- a/src/you_get/extractors/coub.py
+++ b/src/you_get/extractors/coub.py
@@ -58,7 +58,7 @@ def fix_coub_video_file(file_path):
 
 
 def get_title_and_urls(json_data):
-    title = legitimize(re.sub('[\s*]', "_", json_data['title']))
+    title = legitimize(re.sub(r'[\s*]', "_", json_data['title']))
     video_info = json_data['file_versions']['html5']['video']
     if 'high' not in video_info:
         if 'med' not in video_info:
diff --git a/src/you_get/extractors/douban.py b/src/you_get/extractors/douban.py
index 1a4a67d179..17b7a8c591 100644
--- a/src/you_get/extractors/douban.py
+++ b/src/you_get/extractors/douban.py
@@ -10,7 +10,7 @@ def douban_download(url, output_dir = '.', merge = True, info_only = False, **kw
 
     if re.match(r'https?://movie', url):
         title = match1(html, 'name="description" content="([^"]+)')
-        tid = match1(url, 'trailer/(\d+)')
+        tid = match1(url, r'trailer/(\d+)')
         real_url = 'https://movie.douban.com/trailer/video_url?tid=%s' % tid
         type, ext, size = url_info(real_url)
 
diff --git a/src/you_get/extractors/embed.py b/src/you_get/extractors/embed.py
index aedf5137a2..48092df182 100644
--- a/src/you_get/extractors/embed.py
+++ b/src/you_get/extractors/embed.py
@@ -20,18 +20,18 @@
 """
 refer to http://open.youku.com/tools
 """
-youku_embed_patterns = [ 'youku\.com/v_show/id_([a-zA-Z0-9=]+)',
-                         'player\.youku\.com/player\.php/sid/([a-zA-Z0-9=]+)/v\.swf',
-                         'loader\.swf\?VideoIDS=([a-zA-Z0-9=]+)',
-                         'player\.youku\.com/embed/([a-zA-Z0-9=]+)',
-                         'YKU.Player\(\'[a-zA-Z0-9]+\',{ client_id: \'[a-zA-Z0-9]+\', vid: \'([a-zA-Z0-9]+)\''
+youku_embed_patterns = [ r'youku\.com/v_show/id_([a-zA-Z0-9=]+)',
+                         r'player\.youku\.com/player\.php/sid/([a-zA-Z0-9=]+)/v\.swf',
+                         r'loader\.swf\?VideoIDS=([a-zA-Z0-9=]+)',
+                         r'player\.youku\.com/embed/([a-zA-Z0-9=]+)',
+                         r'YKU.Player\(\'[a-zA-Z0-9]+\',{ client_id: \'[a-zA-Z0-9]+\', vid: \'([a-zA-Z0-9]+)\''
                        ]
 
 """
 http://www.tudou.com/programs/view/html5embed.action?type=0&amp;code=3LS_URGvl54&amp;lcode=&amp;resourceId=0_06_05_99
 """
-tudou_embed_patterns = [ 'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_-]+)\&',
-                         'www\.tudou\.com/v/([a-zA-Z0-9_-]+)/[^"]*v\.swf'
+tudou_embed_patterns = [ r'tudou\.com[a-zA-Z0-9\/\?=\&\.\;]+code=([a-zA-Z0-9_-]+)\&',
+                         r'www\.tudou\.com/v/([a-zA-Z0-9_-]+)/[^"]*v\.swf'
                        ]
 
 """
@@ -39,18 +39,18 @@
 """
 tudou_api_patterns = [ ]
 
-iqiyi_embed_patterns = [ 'player\.video\.qiyi\.com/([^/]+)/[^/]+/[^/]+/[^/]+\.swf[^"]+tvId=(\d+)' ]
+iqiyi_embed_patterns = [ r'player\.video\.qiyi\.com/([^/]+)/[^/]+/[^/]+/[^/]+\.swf[^"]+tvId=(\d+)' ]
 
-netease_embed_patterns = [ '(http://\w+\.163\.com/movie/[^\'"]+)' ]
+netease_embed_patterns = [ r'(http://\w+\.163\.com/movie/[^\'"]+)' ]
 
-vimeo_embed_patters = [ 'player\.vimeo\.com/video/(\d+)' ]
+vimeo_embed_patters = [ r'player\.vimeo\.com/video/(\d+)' ]
 
-dailymotion_embed_patterns = [ 'www\.dailymotion\.com/embed/video/(\w+)' ]
+dailymotion_embed_patterns = [ r'www\.dailymotion\.com/embed/video/(\w+)' ]
 
 """
 check the share button on http://www.bilibili.com/video/av5079467/
 """
-bilibili_embed_patterns = [ 'static\.hdslb\.com/miniloader\.swf.*aid=(\d+)' ]
+bilibili_embed_patterns = [ r'static\.hdslb\.com/miniloader\.swf.*aid=(\d+)' ]
 
 
 '''
diff --git a/src/you_get/extractors/funshion.py b/src/you_get/extractors/funshion.py
index 1f7156b5e3..2bb1aba027 100644
--- a/src/you_get/extractors/funshion.py
+++ b/src/you_get/extractors/funshion.py
@@ -84,7 +84,7 @@ def search_dict(a_dict, target):
 
                 moz_ec_name = search_dict(sym_to_name, 'mozEcName')
                 push = search_dict(sym_to_name, 'push')
-                patt = '{}\.{}\("(.+?)"\)'.format(moz_ec_name, push)
+                patt = r'{}\.{}\("(.+?)"\)'.format(moz_ec_name, push)
                 ec_list = re.findall(patt, code)
                 [magic_list.append(sym_to_name[ec]) for ec in ec_list]
         return magic_list
diff --git a/src/you_get/extractors/ku6.py b/src/you_get/extractors/ku6.py
index c827eafd81..d37d894725 100644
--- a/src/you_get/extractors/ku6.py
+++ b/src/you_get/extractors/ku6.py
@@ -50,7 +50,7 @@ def ku6_download(url, output_dir = '.', merge = True, info_only = False, **kwarg
             vid = vid.group(1)
         else:
             raise Exception('Unsupported url')
-        this_meta = re.search('"?'+vid+'"?:\{(.+?)\}', meta)
+        this_meta = re.search('"?'+vid+r'"?:\{(.+?)\}', meta)
         if this_meta is not None:
             this_meta = this_meta.group(1)
             title = re.search('title:"(.+?)"', this_meta).group(1)
diff --git a/src/you_get/extractors/kugou.py b/src/you_get/extractors/kugou.py
index 192bd809d7..18b342bf7e 100644
--- a/src/you_get/extractors/kugou.py
+++ b/src/you_get/extractors/kugou.py
@@ -32,8 +32,8 @@ def kugou_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
 def kugou_download_by_hash(url, output_dir='.', merge=True, info_only=False):
     # sample
     # url_sample:http://www.kugou.com/song/#hash=93F7D2FC6E95424739448218B591AEAF&album_id=9019462
-    hash_val = match1(url, 'hash=(\w+)')
-    album_id = match1(url, 'album_id=(\d+)')
+    hash_val = match1(url, r'hash=(\w+)')
+    album_id = match1(url, r'album_id=(\d+)')
     if not album_id:
         album_id = 123
     html = get_html("http://www.kugou.com/yy/index.php?r=play/getdata&hash={}&album_id={}&mid=123".format(hash_val, album_id))
@@ -60,7 +60,7 @@ def kugou_download_playlist(url, output_dir='.', merge=True, info_only=False, **
         res = pattern.findall(html)
         for song in res:
             res = get_html(song)
-            pattern_url = re.compile('"hash":"(\w+)".*"album_id":(\d)+')
+            pattern_url = re.compile(r'"hash":"(\w+)".*"album_id":(\d)+')
             hash_val, album_id = res = pattern_url.findall(res)[0]
             if not album_id:
                 album_id = 123
@@ -70,7 +70,7 @@ def kugou_download_playlist(url, output_dir='.', merge=True, info_only=False, **
     # album sample:   http://www.kugou.com/yy/album/single/1645030.html
     elif url.lower().find('album') != -1:
         html = get_html(url)
-        pattern = re.compile('var data=(\[.*?\]);')
+        pattern = re.compile(r'var data=(\[.*?\]);')
         res = pattern.findall(html)[0]
         for v in json.loads(res):
             urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v['hash'], v['album_id']))
@@ -79,7 +79,7 @@ def kugou_download_playlist(url, output_dir='.', merge=True, info_only=False, **
     # playlist sample:http://www.kugou.com/yy/special/single/487279.html
     else:
         html = get_html(url)
-        pattern = re.compile('data="(\w+)\|(\d+)"')
+        pattern = re.compile(r'data="(\w+)\|(\d+)"')
         for v in pattern.findall(html):
             urls.append('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v[0], v[1]))
             print('http://www.kugou.com/song/#hash=%s&album_id=%s' % (v[0], v[1]))
diff --git a/src/you_get/extractors/kuwo.py b/src/you_get/extractors/kuwo.py
index 54c09235e9..dceab68fd5 100644
--- a/src/you_get/extractors/kuwo.py
+++ b/src/you_get/extractors/kuwo.py
@@ -18,7 +18,7 @@ def kuwo_download_by_rid(rid, output_dir = '.', merge = True, info_only = False)
 
 def kuwo_playlist_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     html=get_content(url)
-    matched=set(re.compile("yinyue/(\d+)").findall(html))#reduce duplicated
+    matched=set(re.compile(r"yinyue/(\d+)").findall(html))#reduce duplicated
     for rid in matched:
         kuwo_download_by_rid(rid,output_dir,merge,info_only)
 
@@ -26,7 +26,7 @@ def kuwo_playlist_download(url, output_dir = '.', merge = True, info_only = Fals
 
 def kuwo_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     if "www.kuwo.cn/yinyue" in url:
-        rid=match1(url,'yinyue/(\d+)')
+        rid=match1(url, r'yinyue/(\d+)')
         kuwo_download_by_rid(rid,output_dir, merge, info_only)
     else:
         kuwo_playlist_download(url,output_dir,merge,info_only)
diff --git a/src/you_get/extractors/mgtv.py b/src/you_get/extractors/mgtv.py
index 053212efa0..f241ba6f1a 100644
--- a/src/you_get/extractors/mgtv.py
+++ b/src/you_get/extractors/mgtv.py
@@ -44,11 +44,11 @@ def tk2(self):
     def get_vid_from_url(url):
         """Extracts video ID from URL.
         """
-        vid = match1(url, 'https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
+        vid = match1(url, r'https?://www.mgtv.com/(?:b|l)/\d+/(\d+).html')
         if not vid:
-            vid = match1(url, 'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
+            vid = match1(url, r'https?://www.mgtv.com/hz/bdpz/\d+/(\d+).html')
         if not vid:
-            vid = match1(url, 'https?://www.mgtv.com/s/(\d+).html')
+            vid = match1(url, r'https?://www.mgtv.com/s/(\d+).html')
         return vid
 
     # ----------------------------------------------------------------------
diff --git a/src/you_get/extractors/qq.py b/src/you_get/extractors/qq.py
index e38770e9c0..6cb76e1d9c 100644
--- a/src/you_get/extractors/qq.py
+++ b/src/you_get/extractors/qq.py
@@ -83,7 +83,7 @@ def kg_qq_download_by_shareid(shareid, output_dir='.', info_only=False, caption=
     playurl = json_data['data']['playurl']
     videourl = json_data['data']['playurl_video']
     real_url = playurl if playurl else videourl
-    real_url = real_url.replace('\/', '/')
+    real_url = real_url.replace(r'\/', '/')
 
     ksong_mid = json_data['data']['ksong_mid']
     lyric_url = 'http://cgi.kg.qq.com/fcgi-bin/fcg_lyric?jsonpCallback=jsopgetlrcdata&outCharset=utf-8&ksongmid=' + ksong_mid
diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index 74374202e2..e0e4eeb97e 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -23,7 +23,7 @@ def real_url(fileName, key, ch):
 
 def sohu_download(url, output_dir='.', merge=True, info_only=False, extractor_proxy=None, **kwargs):
     if re.match(r'http://share.vrs.sohu.com', url):
-        vid = r1('id=(\d+)', url)
+        vid = r1(r'id=(\d+)', url)
     else:
         html = get_html(url)
         vid = r1(r'\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?', html) or r1(r'bid:\'(\d+)\',', html) or r1(r'bid=(\d+)', html)
diff --git a/src/you_get/extractors/tudou.py b/src/you_get/extractors/tudou.py
index b1568dfd13..92b8393ce6 100644
--- a/src/you_get/extractors/tudou.py
+++ b/src/you_get/extractors/tudou.py
@@ -71,7 +71,7 @@ def tudou_download(url, output_dir = '.', merge = True, info_only = False, **kwa
 
 # obsolete?
 def parse_playlist(url):
-    aid = r1('http://www.tudou.com/playlist/p/a(\d+)(?:i\d+)?\.html', url)
+    aid = r1(r'http://www.tudou.com/playlist/p/a(\d+)(?:i\d+)?\.html', url)
     html = get_decoded_html(url)
     if not aid:
         aid = r1(r"aid\s*[:=]\s*'(\d+)'", html)
diff --git a/src/you_get/extractors/tumblr.py b/src/you_get/extractors/tumblr.py
index 08e605e82c..236ba18274 100644
--- a/src/you_get/extractors/tumblr.py
+++ b/src/you_get/extractors/tumblr.py
@@ -34,7 +34,7 @@ def tumblr_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
                      post_data_raw='{"eu_resident":true,"gdpr_is_acceptable_age":true,"gdpr_consent_core":true,"gdpr_consent_first_party_ads":true,"gdpr_consent_third_party_ads":true,"gdpr_consent_search_history":true,"redirect_to":"%s","gdpr_reconsent":false}' % url)
         page = get_html(url, faker=True)
 
-    html = parse.unquote(page).replace('\/', '/')
+    html = parse.unquote(page).replace(r'\/', '/')
     feed = r1(r'<meta property="og:type" content="tumblr-feed:(\w+)" />', html)
 
     if feed in ['photo', 'photoset', 'entry'] or feed is None:
diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index b2c5878a9d..299dc05240 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -33,7 +33,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
                              **kwargs)
         return
 
-    m = re.match('^https?://(mobile\.)?(x|twitter)\.com/([^/]+)/status/(\d+)', url)
+    m = re.match(r'^https?://(mobile\.)?(x|twitter)\.com/([^/]+)/status/(\d+)', url)
     assert m
     screen_name, item_id = m.group(3), m.group(4)
     page_title = "{} [{}]".format(screen_name, item_id)
diff --git a/src/you_get/extractors/universal.py b/src/you_get/extractors/universal.py
index 4a3268ab89..03bba35af6 100644
--- a/src/you_get/extractors/universal.py
+++ b/src/you_get/extractors/universal.py
@@ -48,7 +48,7 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
             else:
                 return
 
-        hls_urls = re.findall(r'(https?://[^;"\'\\]+' + '\.m3u8?' +
+        hls_urls = re.findall(r'(https?://[^;"\'\\]+' + r'\.m3u8?' +
                               r'[^;"\'\\]*)', page)
         if hls_urls:
             try:
@@ -64,14 +64,14 @@ def universal_download(url, output_dir='.', merge=True, info_only=False, **kwarg
                 return
 
         # most common media file extensions on the Internet
-        media_exts = ['\.flv', '\.mp3', '\.mp4', '\.webm',
-                      '[-_]1\d\d\d\.jpe?g', '[-_][6-9]\d\d\.jpe?g', # tumblr
-                      '[-_]1\d\d\dx[6-9]\d\d\.jpe?g',
-                      '[-_][6-9]\d\dx1\d\d\d\.jpe?g',
-                      '[-_][6-9]\d\dx[6-9]\d\d\.jpe?g',
-                      's1600/[\w%]+\.jpe?g', # blogger
-                      'blogger\.googleusercontent\.com/img/a/\w*', # blogger
-                      'img[6-9]\d\d/[\w%]+\.jpe?g' # oricon?
+        media_exts = [r'\.flv', r'\.mp3', r'\.mp4', r'\.webm',
+                      r'[-_]1\d\d\d\.jpe?g', r'[-_][6-9]\d\d\.jpe?g', # tumblr
+                      r'[-_]1\d\d\dx[6-9]\d\d\.jpe?g',
+                      r'[-_][6-9]\d\dx1\d\d\d\.jpe?g',
+                      r'[-_][6-9]\d\dx[6-9]\d\d\.jpe?g',
+                      r's1600/[\w%]+\.jpe?g', # blogger
+                      r'blogger\.googleusercontent\.com/img/a/\w*', # blogger
+                      r'img[6-9]\d\d/[\w%]+\.jpe?g' # oricon?
         ]
 
         urls = []
diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index c7d7b05754..8c9f8ebc9e 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -102,7 +102,7 @@ def extract(self, **kwargs):
         pos = 0
         while pos < len(lines):
             if lines[pos].startswith('#EXT-X-STREAM-INF'):
-                patt = 'RESOLUTION=(\d+)x(\d+)'
+                patt = r'RESOLUTION=(\d+)x(\d+)'
                 hit = re.search(patt, lines[pos])
                 if hit is None:
                     continue
diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index ed0743bbc0..7a6fb2fc99 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -242,7 +242,7 @@ def prepare(self, **kwargs):
 
 def youku_download_playlist_by_url(url, **kwargs):
     video_page_pt = 'https?://v.youku.com/v_show/id_([A-Za-z0-9=]+)'
-    js_cb_pt = '\(({.+})\)'
+    js_cb_pt = r'\(({.+})\)'
     if re.match(video_page_pt, url):
         youku_obj = Youku()
         youku_obj.url = url
@@ -272,14 +272,14 @@ def youku_download_playlist_by_url(url, **kwargs):
         page = get_content(url)
         show_id = re.search(r'showid:"(\d+)"', page).group(1)
         ep = 'http://list.youku.com/show/module?id={}&tab=showInfo&callback=jQuery'.format(show_id)
-        xhr_page = get_content(ep).replace('\/', '/').replace('\"', '"')
+        xhr_page = get_content(ep).replace(r'\/', '/').replace(r'\"', '"')
         video_url = re.search(r'(v.youku.com/v_show/id_(?:[A-Za-z0-9=]+)\.html)', xhr_page).group(1)
         youku_download_playlist_by_url('http://'+video_url, **kwargs)
         return
-    elif re.match('https?://list.youku.com/albumlist/show/id_(\d+)\.html', url):
+    elif re.match(r'https?://list.youku.com/albumlist/show/id_(\d+)\.html', url):
         # http://list.youku.com/albumlist/show/id_2336634.html
         # UGC playlist
-        list_id = re.search('https?://list.youku.com/albumlist/show/id_(\d+)\.html', url).group(1)
+        list_id = re.search(r'https?://list.youku.com/albumlist/show/id_(\d+)\.html', url).group(1)
         ep = 'http://list.youku.com/albumlist/items?id={}&page={}&size=20&ascending=1&callback=tuijsonp6'
 
         first_u = ep.format(list_id, 1)
@@ -294,7 +294,7 @@ def youku_download_playlist_by_url(url, **kwargs):
             for i in range(2, req_cnt+2):
                 req_u = ep.format(list_id, i)
                 xhr_page = get_content(req_u)
-                json_data = json.loads(re.search(js_cb_pt, xhr_page).group(1).replace('\/', '/'))
+                json_data = json.loads(re.search(js_cb_pt, xhr_page).group(1).replace(r'\/', '/'))
                 xhr_html = json_data['html']
                 page_videos = re.findall(r'(v.youku.com/v_show/id_(?:[A-Za-z0-9=]+)\.html)', xhr_html)
                 v_urls.extend(page_videos)
diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 0a93d396b8..fe06419940 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -197,7 +197,7 @@ def prepare(self, **kwargs):
                 self.download_playlist_by_url(self.url, **kwargs)
                 exit(0)
 
-        if re.search('\Wlist=', self.url) and not kwargs.get('playlist'):
+        if re.search(r'\Wlist=', self.url) and not kwargs.get('playlist'):
             log.w('This video is from a playlist. (use --playlist to download all videos in the playlist.)')
 
         # Extract from video page
@@ -205,7 +205,7 @@ def prepare(self, **kwargs):
         video_page = get_content('https://www.youtube.com/watch?v=%s' % self.vid, headers={'User-Agent': self.ua})
 
         try:
-            jsUrl = re.search('([^"]*/base\.js)"', video_page).group(1)
+            jsUrl = re.search(r'([^"]*/base\.js)"', video_page).group(1)
         except:
             log.wtf('[Failed] Unable to find base.js on the video page')
         self.html5player = 'https://www.youtube.com' + jsUrl
@@ -213,7 +213,7 @@ def prepare(self, **kwargs):
         self.js = get_content(self.html5player).replace('\n', ' ')
 
         logging.debug('Loading ytInitialPlayerResponse...')
-        ytInitialPlayerResponse = json.loads(re.search('ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>|var )', video_page).group(1))
+        ytInitialPlayerResponse = json.loads(re.search(r'ytInitialPlayerResponse\s*=\s*([^\n]+?});(\n|</script>|var )', video_page).group(1))
         self.check_playability_response(ytInitialPlayerResponse)
 
         # Get the video title

From 26db833b86e13f83acc00367533e07a35ab66e49 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 19 Aug 2024 18:41:21 +0200
Subject: [PATCH 1208/1225] [tests] disable test_youtube temporarily

---
 tests/test.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/test.py b/tests/test.py
index f04b1fe236..2d220c6214 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -27,18 +27,18 @@ def test_magisto(self):
             info_only=True
         )
 
-    def test_youtube(self):
-        youtube.download(
-            'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True
-        )
+    #def test_youtube(self):
+        #youtube.download(
+        #    'http://www.youtube.com/watch?v=pzKerr0JIPA', info_only=True
+        #)
         #youtube.download('http://youtu.be/pzKerr0JIPA', info_only=True)
         #youtube.download(
         #    'http://www.youtube.com/attribution_link?u=/watch?v%3DldAKIzq7bvs%26feature%3Dshare',  # noqa
         #    info_only=True
         #)
-        youtube.download(
-            'https://www.youtube.com/watch?v=oRdxUFDoQe0', info_only=True
-        )
+        #youtube.download(
+        #    'https://www.youtube.com/watch?v=oRdxUFDoQe0', info_only=True
+        #)
 
     def test_acfun(self):
         acfun.download('https://www.acfun.cn/v/ac44560432', info_only=True)

From 2c8a0cb5e87692ce3132d284cd5f71cb50006cfe Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 19 Aug 2024 21:48:34 +0200
Subject: [PATCH 1209/1225] [vimeo] cleanup

---
 src/you_get/extractors/vimeo.py | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/src/you_get/extractors/vimeo.py b/src/you_get/extractors/vimeo.py
index 8c9f8ebc9e..4034d0e054 100644
--- a/src/you_get/extractors/vimeo.py
+++ b/src/you_get/extractors/vimeo.py
@@ -132,34 +132,6 @@ def extract(self, **kwargs):
 
 
 def vimeo_download_by_id(id, title=None, output_dir='.', merge=True, info_only=False, **kwargs):
-    '''
-    try:
-        # normal Vimeo video
-        html = get_content('https://vimeo.com/' + id)
-        cfg_patt = r'clip_page_config\s*=\s*(\{.+?\});'
-        cfg = json.loads(match1(html, cfg_patt))
-        video_page = get_content(cfg['player']['config_url'], headers=fake_headers)
-        title = cfg['clip']['title']
-        info = loads(video_page)
-    except:
-        # embedded player - referer may be required
-        if 'referer' in kwargs:
-            fake_headers['Referer'] = kwargs['referer']
-
-        video_page = get_content('http://player.vimeo.com/video/%s' % id, headers=fake_headers)
-        title = r1(r'<title>([^<]+)</title>', video_page)
-        info = loads(match1(video_page, r'var t=(\{.+?\});'))
-
-    streams = info['request']['files']['progressive']
-    streams = sorted(streams, key=lambda i: i['height'])
-    url = streams[-1]['url']
-
-    type, ext, size = url_info(url, faker=True)
-
-    print_info(site_info, title, type, size)
-    if not info_only:
-        download_urls([url], title, ext, size, output_dir, merge=merge, faker=True)
-    '''
     site = VimeoExtractor()
     site.download_by_vid(id, info_only=info_only, output_dir=output_dir, merge=merge, **kwargs)
 

From c4042d0d0bba7b6662bd55a5767e6f1562f43341 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 19 Aug 2024 21:53:12 +0200
Subject: [PATCH 1210/1225] version 0.4.1730

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index 742f2def02..e006820829 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1718'
+__version__ = '0.4.1730'

From 585d496e16881115d5af443cb6d60a293f9905e0 Mon Sep 17 00:00:00 2001
From: doskey <doskey.lee@gmail.com>
Date: Tue, 10 Sep 2024 00:13:51 +0800
Subject: [PATCH 1211/1225] Fix: Restore functionality of -o parameter for
 Miaopai downloads

The -o parameter, used for specifying output filename, was not functioning correctly when downloading videos from Miaopai. This commit addresses the issue and ensures the parameter works as intended.
---
 src/you_get/extractors/miaopai.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/you_get/extractors/miaopai.py b/src/you_get/extractors/miaopai.py
index 91a301b832..63c8685c22 100644
--- a/src/you_get/extractors/miaopai.py
+++ b/src/you_get/extractors/miaopai.py
@@ -37,7 +37,7 @@ def miaopai_download_by_fid(fid, output_dir = '.', merge = False, info_only = Fa
         download_urls([url], title, ext, total_size=None, output_dir=output_dir, merge=merge)
 
 
-def miaopai_download_by_wbmp(wbmp_url, fid, info_only=False, **kwargs):
+def miaopai_download_by_wbmp(wbmp_url, fid, output_dir='.', merge=False, info_only=False, **kwargs):
     headers = {}
     headers.update(fake_headers_mobile)
     headers['Host'] = 'imgaliyuncdn.miaopai.com'
@@ -62,7 +62,7 @@ def miaopai_download_by_wbmp(wbmp_url, fid, info_only=False, **kwargs):
     headers['Host'] = 'f.us.sinaimg.cn'
     print_info(site_info, title, ext, url_info(video_url, headers=headers)[2])
     if not info_only:
-        download_urls([video_url], fs.legitimize(title), ext, headers=headers, **kwargs)
+        download_urls([video_url], fs.legitimize(title), ext, output_dir=output_dir, headers=headers, **kwargs)
 
 
 def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **kwargs):
@@ -75,7 +75,7 @@ def miaopai_download_story(url, output_dir='.', merge=False, info_only=False, **
     ext = 'mp4'
     print_info(site_info, title, ext, url_info(stream_url, headers=fake_headers_mobile)[2])
     if not info_only:
-        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
+        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, output_dir=output_dir, headers=fake_headers_mobile, **kwargs)
 
 
 def miaopai_download_h5api(url, output_dir='.', merge=False, info_only=False, **kwargs):
@@ -122,7 +122,7 @@ def miaopai_download_h5api(url, output_dir='.', merge=False, info_only=False, **
 
     print_info(site_info, title, ext, size)
     if not info_only:
-        download_urls([video_url], fs.legitimize(title), ext, total_size=size, headers=headers, **kwargs)
+        download_urls([video_url], fs.legitimize(title), ext, total_size=size, output_dir=output_dir, headers=headers, **kwargs)
 
 
 def miaopai_download_direct(url, output_dir='.', merge=False, info_only=False, **kwargs):
@@ -141,7 +141,7 @@ def miaopai_download_direct(url, output_dir='.', merge=False, info_only=False, *
     ext = 'mp4'
     print_info(site_info, title, ext, url_info(stream_url, headers=fake_headers_mobile)[2])
     if not info_only:
-        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, headers=fake_headers_mobile, **kwargs)
+        download_urls([stream_url], fs.legitimize(title), ext, total_size=None, output_dir=output_dir, headers=fake_headers_mobile, **kwargs)
 
 
 def miaopai_download(url, output_dir='.', merge=False, info_only=False, **kwargs):

From 094d5a074c950733d7e7cddfd8dadfcbba3b15b6 Mon Sep 17 00:00:00 2001
From: lcjh <120989324@qq.com>
Date: Sat, 12 Oct 2024 03:21:23 +0800
Subject: [PATCH 1212/1225] remove imported but unused module

---
 src/you_get/extractors/acfun.py      | 4 ++--
 src/you_get/extractors/baidu.py      | 4 ++--
 src/you_get/extractors/facebook.py   | 1 -
 src/you_get/extractors/fc2video.py   | 1 -
 src/you_get/extractors/giphy.py      | 1 -
 src/you_get/extractors/iqiyi.py      | 1 -
 src/you_get/extractors/ixigua.py     | 5 -----
 src/you_get/extractors/kuaishou.py   | 2 --
 src/you_get/extractors/kugou.py      | 1 -
 src/you_get/extractors/pixnet.py     | 1 -
 src/you_get/extractors/sohu.py       | 3 ---
 src/you_get/extractors/soundcloud.py | 1 -
 src/you_get/extractors/suntv.py      | 1 -
 src/you_get/extractors/ucas.py       | 1 -
 src/you_get/extractors/veoh.py       | 1 -
 src/you_get/extractors/yizhibo.py    | 1 -
 src/you_get/extractors/youku.py      | 1 -
 17 files changed, 4 insertions(+), 26 deletions(-)

diff --git a/src/you_get/extractors/acfun.py b/src/you_get/extractors/acfun.py
index 5775eb5e74..e9ecad3fdb 100644
--- a/src/you_get/extractors/acfun.py
+++ b/src/you_get/extractors/acfun.py
@@ -43,7 +43,7 @@ def prepare(self, **kwargs):
             currentVideoInfo = json_data.get('currentVideoInfo')
 
         else:
-            raise NotImplemented            
+            raise NotImplementedError()         
 
         if 'ksPlayJson' in currentVideoInfo:
             durationMillis = currentVideoInfo['durationMillis']
@@ -193,7 +193,7 @@ def getM3u8UrlFromCurrentVideoInfo(currentVideoInfo):
             m3u8_url = getM3u8UrlFromCurrentVideoInfo(currentVideoInfo)
 
         else:
-            raise NotImplemented
+            raise NotImplementedError()
 
         assert title and m3u8_url
         title = unescape_html(title)
diff --git a/src/you_get/extractors/baidu.py b/src/you_get/extractors/baidu.py
index 61b0ad246f..4f64f34def 100644
--- a/src/you_get/extractors/baidu.py
+++ b/src/you_get/extractors/baidu.py
@@ -185,7 +185,7 @@ def baidu_pan_download(url):
     isprotected = False
     sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
         html)
-    if sign == None:
+    if sign is None:
         if re.findall(r'\baccess-code\b', html):
             isprotected = True
             sign, timestamp, bdstoken, appid, primary_id, fs_id, uk, fake_headers, psk = baidu_pan_protected_share(
@@ -193,7 +193,7 @@ def baidu_pan_download(url):
             # raise NotImplementedError("Password required!")
         if isprotected != True:
             raise AssertionError("Share not found or canceled: %s" % url)
-    if bdstoken == None:
+    if bdstoken is None:
         bdstoken = ""
     if isprotected != True:
         sign, timestamp, bdstoken, appid, primary_id, fs_id, uk = baidu_pan_parse(
diff --git a/src/you_get/extractors/facebook.py b/src/you_get/extractors/facebook.py
index ba8aea8c72..4940390734 100644
--- a/src/you_get/extractors/facebook.py
+++ b/src/you_get/extractors/facebook.py
@@ -3,7 +3,6 @@
 __all__ = ['facebook_download']
 
 from ..common import *
-import json
 
 def facebook_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     url = re.sub(r'//.*?facebook.com','//facebook.com',url)
diff --git a/src/you_get/extractors/fc2video.py b/src/you_get/extractors/fc2video.py
index d6f522ce61..22efcee404 100644
--- a/src/you_get/extractors/fc2video.py
+++ b/src/you_get/extractors/fc2video.py
@@ -5,7 +5,6 @@
 from ..common import *
 from hashlib import md5
 from urllib.parse import urlparse
-import re
 
 #----------------------------------------------------------------------
 def makeMimi(upid):
diff --git a/src/you_get/extractors/giphy.py b/src/you_get/extractors/giphy.py
index 1dd30223d2..8bbba0f0bb 100644
--- a/src/you_get/extractors/giphy.py
+++ b/src/you_get/extractors/giphy.py
@@ -3,7 +3,6 @@
 __all__ = ['giphy_download']
 
 from ..common import *
-import json
 
 def giphy_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
     html = get_html(url)
diff --git a/src/you_get/extractors/iqiyi.py b/src/you_get/extractors/iqiyi.py
index 16bf45d3a3..cfd805fefe 100644
--- a/src/you_get/extractors/iqiyi.py
+++ b/src/you_get/extractors/iqiyi.py
@@ -10,7 +10,6 @@
 from random import random,randint
 import json
 from math import floor
-from zlib import decompress
 import hashlib
 import time
 
diff --git a/src/you_get/extractors/ixigua.py b/src/you_get/extractors/ixigua.py
index f2fd953e69..a6b3c2d80a 100644
--- a/src/you_get/extractors/ixigua.py
+++ b/src/you_get/extractors/ixigua.py
@@ -1,12 +1,7 @@
 #!/usr/bin/env python
 import base64
 
-import binascii
-
 from ..common import *
-import random
-import string
-import ctypes
 from json import loads
 from urllib import request
 
diff --git a/src/you_get/extractors/kuaishou.py b/src/you_get/extractors/kuaishou.py
index 917920d196..d2ce8709d2 100644
--- a/src/you_get/extractors/kuaishou.py
+++ b/src/you_get/extractors/kuaishou.py
@@ -2,10 +2,8 @@
 
 import urllib.request
 import urllib.parse
-import json
 import re
 
-from ..util import log
 from ..common import get_content, download_urls, print_info, playlist_not_supported, url_size
 
 __all__ = ['kuaishou_download_by_url']
diff --git a/src/you_get/extractors/kugou.py b/src/you_get/extractors/kugou.py
index 18b342bf7e..1b6df53f1b 100644
--- a/src/you_get/extractors/kugou.py
+++ b/src/you_get/extractors/kugou.py
@@ -6,7 +6,6 @@
 from json import loads
 from base64 import b64decode
 import re
-import hashlib
 
 
 def kugou_download(url, output_dir=".", merge=True, info_only=False, **kwargs):
diff --git a/src/you_get/extractors/pixnet.py b/src/you_get/extractors/pixnet.py
index b5c94b2428..85304c0c91 100644
--- a/src/you_get/extractors/pixnet.py
+++ b/src/you_get/extractors/pixnet.py
@@ -3,7 +3,6 @@
 __all__ = ['pixnet_download']
 
 from ..common import *
-import urllib.error
 from time import time
 from urllib.parse import quote
 from json import loads
diff --git a/src/you_get/extractors/sohu.py b/src/you_get/extractors/sohu.py
index e0e4eeb97e..489f0737e7 100644
--- a/src/you_get/extractors/sohu.py
+++ b/src/you_get/extractors/sohu.py
@@ -5,9 +5,6 @@
 from ..common import *
 
 import json
-import time
-from random import random
-from urllib.parse import urlparse
 
 '''
 Changelog:
diff --git a/src/you_get/extractors/soundcloud.py b/src/you_get/extractors/soundcloud.py
index 08e9d5610e..a0f2d2da47 100644
--- a/src/you_get/extractors/soundcloud.py
+++ b/src/you_get/extractors/soundcloud.py
@@ -5,7 +5,6 @@
 from ..common import *
 import re
 import json
-import urllib.error
 
 
 def get_sndcd_apikey():
diff --git a/src/you_get/extractors/suntv.py b/src/you_get/extractors/suntv.py
index 0b50644038..25d065eccc 100644
--- a/src/you_get/extractors/suntv.py
+++ b/src/you_get/extractors/suntv.py
@@ -27,7 +27,6 @@ def suntv_download(url, output_dir = '.', merge = True, info_only = False, **kwa
         html = html.decode('gbk')
         title = match1(html, '<title>([^<]+)').strip()  #get rid of \r\n s
         
-        type_ = ''
         size = 0
         type, ext, size = url_info(video_url)
         
diff --git a/src/you_get/extractors/ucas.py b/src/you_get/extractors/ucas.py
index 4f07bce5a0..37988c6605 100644
--- a/src/you_get/extractors/ucas.py
+++ b/src/you_get/extractors/ucas.py
@@ -3,7 +3,6 @@
 __all__ = ['ucas_download', 'ucas_download_single', 'ucas_download_playlist']
 
 from ..common import *
-import urllib.error
 import http.client
 from time import time
 from random import random
diff --git a/src/you_get/extractors/veoh.py b/src/you_get/extractors/veoh.py
index eb37c1ebb2..4c45456b80 100644
--- a/src/you_get/extractors/veoh.py
+++ b/src/you_get/extractors/veoh.py
@@ -3,7 +3,6 @@
 __all__ = ['veoh_download']
 
 from ..common import *
-import urllib.error
 
 def veoh_download(url, output_dir = '.', merge = False, info_only = False, **kwargs):
     '''Get item_id'''
diff --git a/src/you_get/extractors/yizhibo.py b/src/you_get/extractors/yizhibo.py
index 11ce86ad3b..e6e590ffdb 100644
--- a/src/you_get/extractors/yizhibo.py
+++ b/src/you_get/extractors/yizhibo.py
@@ -4,7 +4,6 @@
 
 from ..common import *
 import json
-import time
 
 def yizhibo_download(url, output_dir = '.', merge = True, info_only = False, **kwargs):
     video_id = url[url.rfind('/')+1:].split(".")[0]
diff --git a/src/you_get/extractors/youku.py b/src/you_get/extractors/youku.py
index 7a6fb2fc99..d34571afe8 100644
--- a/src/you_get/extractors/youku.py
+++ b/src/you_get/extractors/youku.py
@@ -5,7 +5,6 @@
 from ..extractor import VideoExtractor
 
 import time
-import traceback
 import json
 import urllib.request
 import urllib.parse

From 1c9c0f3ed1b8466239fa8656523658ccce8bb489 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Oct 2024 23:23:47 +0100
Subject: [PATCH 1213/1225] [youtube] fix extraction (close #3031)

---
 src/you_get/extractors/youtube.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index fe06419940..23112b2dea 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -82,6 +82,14 @@ def n_to_n(js, n):
             #   Xka - https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/sv_SE/base.js
             #   jma - https://www.youtube.com/s/player/8d9f6215/player_ias.vflset/sv_SE/base.js
             f1 = match1(js, r',[$\w]+\.length\|\|([$\w]+)\(""\)\)}};')
+
+            # Examples:
+            #   Yla, ida - https://www.youtube.com/s/player/fb725ac8/player-plasma-ias-phone-sv_SE.vflset/base.js
+            #   Hla, eda - https://www.youtube.com/s/player/2f238d39/player-plasma-ias-phone-en_US.vflset/base.js
+            if not f1:
+                f0 = match1(js, r'c=([$\w]+)\[0\]\(c\),a\.set\(b,c\)')
+                f1 = match1(js, r'%s=\[([$\w]+)\]' % f0)
+
             f1def = match1(js, r'\W%s=(function\(\w+\).+?\)});' % re.escape(f1))
             n = dukpy.evaljs('(%s)("%s")' % (f1def, n))
             return n
@@ -186,7 +194,7 @@ def check_playability_response(self, ytInitialPlayerResponse):
             )
 
     def prepare(self, **kwargs):
-        self.ua = 'Mozilla/5.0 (Linux; Android 14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.6533.103 Mobile Safari/537.36'
+        self.ua = 'Mozilla/5.0 (Linux; Android 15) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.73 Mobile Safari/537.36'
 
         assert self.url or self.vid
 

From 5b4142dc4a0a42604d799e3088c1696b8c96647e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Oct 2024 23:57:56 +0100
Subject: [PATCH 1214/1225] [twitter] fix extraction

---
 src/you_get/extractors/twitter.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/you_get/extractors/twitter.py b/src/you_get/extractors/twitter.py
index 299dc05240..212386d909 100644
--- a/src/you_get/extractors/twitter.py
+++ b/src/you_get/extractors/twitter.py
@@ -15,6 +15,12 @@ def extract_m3u(source):
     return ['https://video.twimg.com%s' % i for i in s2]
 
 def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs):
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0',
+        'Accept-Encoding': 'gzip, deflate',
+        'Accept': '*/*'
+    }
+
     if re.match(r'https?://pbs\.twimg\.com', url):
         universal_download(url, output_dir, merge=merge, info_only=info_only, **kwargs)
         return
@@ -52,7 +58,7 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             photo_url = photo['url']
             title = item_id + '_' + photo_url.split('.')[-2].split('/')[-1]
             urls = [ photo_url + ':orig' ]
-            size = urls_size(urls)
+            size = urls_size(urls, headers=headers)
             ext = photo_url.split('.')[-1]
 
             print_info(site_info, title, ext, size)
@@ -66,12 +72,12 @@ def twitter_download(url, output_dir='.', merge=True, info_only=False, **kwargs)
             variants = sorted(variants, key=lambda kv: kv.get('bitrate', 0))
             title = item_id + '_' + variants[-1]['url'].split('/')[-1].split('?')[0].split('.')[0]
             urls = [ variants[-1]['url'] ]
-            size = urls_size(urls)
+            size = urls_size(urls, headers=headers)
             mime, ext = variants[-1]['content_type'], 'mp4'
 
             print_info(site_info, title, ext, size)
             if not info_only:
-                download_urls(urls, title, ext, size, output_dir, merge=merge)
+                download_urls(urls, title, ext, size, output_dir, merge=merge, headers=headers)
 
     # TODO: should we deal with quoted tweets?
 

From ab9a9785c0dbc229a0cc359d0062ceb232780862 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Mon, 28 Oct 2024 17:29:05 +0100
Subject: [PATCH 1215/1225] [youtube] fix reason

---
 src/you_get/extractors/youtube.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 23112b2dea..fe43dfa64c 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -188,10 +188,17 @@ def check_playability_response(self, ytInitialPlayerResponse):
 
         playerResponseStatus = ytInitialPlayerResponse["playabilityStatus"]["status"]
         if playerResponseStatus != STATUS_OK:
-            reason = ytInitialPlayerResponse["playabilityStatus"].get("reason", "")
-            raise AssertionError(
-                f"Server refused to provide video details. Returned status: {playerResponseStatus}, reason: {reason}."
-            )
+            try:
+                reason = ytInitialPlayerResponse["playabilityStatus"]['errorScreen']\
+                    ['playerErrorMessageRenderer']['reason']['runs'][0]['text']
+                reason += ' ' + ytInitialPlayerResponse["playabilityStatus"]['errorScreen']\
+                    ['playerErrorMessageRenderer']['subreason']['runs'][0]['text']
+            except:
+                reason = ytInitialPlayerResponse["playabilityStatus"].get("reason", "")
+            if reason:
+                log.wtf(f'Server refused to provide video details. Returned status: {playerResponseStatus}. Reason: {reason}')
+            else:
+                log.wtf(f'Server refused to provide video details. Returned status: {playerResponseStatus}.')
 
     def prepare(self, **kwargs):
         self.ua = 'Mozilla/5.0 (Linux; Android 15) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.73 Mobile Safari/537.36'

From aeb1321af9e70d190fce88dbbd81f985836abb81 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Tue, 10 Dec 2024 11:19:23 +0100
Subject: [PATCH 1216/1225] [youtube] fix extraction (player 3bb1f723)

---
 src/you_get/extractors/youtube.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index fe43dfa64c..ffbc9adadc 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -86,12 +86,17 @@ def n_to_n(js, n):
             # Examples:
             #   Yla, ida - https://www.youtube.com/s/player/fb725ac8/player-plasma-ias-phone-sv_SE.vflset/base.js
             #   Hla, eda - https://www.youtube.com/s/player/2f238d39/player-plasma-ias-phone-en_US.vflset/base.js
+            #   WyE, bE7, Gsn - https://www.youtube.com/s/player/3bb1f723/player-plasma-ias-phone-sv_SE.vflset/base.js
             if not f1:
-                f0 = match1(js, r'c=([$\w]+)\[0\]\(c\),a\.set\(b,c\)')
+                f0 = match1(js, r'\w=([$\w]+)\[0\]\(\w\),\w\.set\(\w,\w\)')
                 f1 = match1(js, r'%s=\[([$\w]+)\]' % f0)
 
             f1def = match1(js, r'\W%s=(function\(\w+\).+?\)});' % re.escape(f1))
-            n = dukpy.evaljs('(%s)("%s")' % (f1def, n))
+            v1 = match1(f1def, r'if\(typeof ([$\w]+)==="undefined"\)')
+            v1def = match1(js, r'(var %s=[^;]+;)' % v1)
+            if not v1def:
+                v1def = ''
+            n = dukpy.evaljs('%s(%s)("%s")' % (v1def, f1def, n))
             return n
 
         u = urlparse(url)

From 7fbd4c3e00cc5bf2842dd55bb206ae2ac8fcc6bb Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Wed, 11 Dec 2024 00:28:31 +0100
Subject: [PATCH 1217/1225] python-package.yml: update artifact actions

---
 .github/workflows/python-package.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 2ee74663d8..70fcc335e6 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,12 +15,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', pypy-3.8, pypy-3.9, pypy-3.10]
+        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.8, pypy-3.9, pypy-3.10]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

From 51a7eb5efccb226576cd0d3a018bc3c1f19a1081 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 29 Dec 2024 12:36:44 +0100
Subject: [PATCH 1218/1225] [youtube] update self.ua (fix extraction)

---
 src/you_get/extractors/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index ffbc9adadc..59f22c3cd2 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -206,7 +206,7 @@ def check_playability_response(self, ytInitialPlayerResponse):
                 log.wtf(f'Server refused to provide video details. Returned status: {playerResponseStatus}.')
 
     def prepare(self, **kwargs):
-        self.ua = 'Mozilla/5.0 (Linux; Android 15) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.6723.73 Mobile Safari/537.36'
+        self.ua = 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)'
 
         assert self.url or self.vid
 

From f25ddca25afb1a779fbb5f884011da1d003b39f2 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 29 Dec 2024 13:07:53 +0100
Subject: [PATCH 1219/1225] [youtube] fix caption tracks extraction

---
 src/you_get/extractors/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/you_get/extractors/youtube.py b/src/you_get/extractors/youtube.py
index 59f22c3cd2..8ecdc8aee3 100644
--- a/src/you_get/extractors/youtube.py
+++ b/src/you_get/extractors/youtube.py
@@ -286,6 +286,8 @@ def prepare(self, **kwargs):
             for ct in caption_tracks:
                 ttsurl, lang = ct['baseUrl'], ct['languageCode']
 
+                if ttsurl.startswith('/'):
+                    ttsurl = 'https://www.youtube.com' + ttsurl
                 tts_xml = parseString(get_content(ttsurl))
                 transcript = tts_xml.getElementsByTagName('transcript')[0]
                 texts = transcript.getElementsByTagName('text')

From e9165e07de315dad5f6b09df8368f2188727a31e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sat, 4 Jan 2025 02:50:35 +0100
Subject: [PATCH 1220/1225] version 0.4.1743

---
 src/you_get/version.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/you_get/version.py b/src/you_get/version.py
index e006820829..79ec51d05f 100644
--- a/src/you_get/version.py
+++ b/src/you_get/version.py
@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 
 script_name = 'you-get'
-__version__ = '0.4.1730'
+__version__ = '0.4.1743'

From 4fb7d23602fd2bd57866c217aba7e54a3fe08e78 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Apr 2025 01:58:30 +0200
Subject: [PATCH 1221/1225] [common] fix a long-standing bug that causes
 infinite downloading when content-length is missing

---
 src/you_get/common.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/you_get/common.py b/src/you_get/common.py
index 0b307ddee8..c99e13da60 100755
--- a/src/you_get/common.py
+++ b/src/you_get/common.py
@@ -807,6 +807,8 @@ def numreturn(a):
                     except socket.timeout:
                         pass
                     if not buffer:
+                        if file_size == float('+inf'):  # Prevent infinite downloading
+                            break
                         if is_chunked and received_chunk == range_length:
                             break
                         elif not is_chunked and received == file_size:  # Download finished
@@ -827,9 +829,10 @@ def numreturn(a):
         received, os.path.getsize(temp_filepath), temp_filepath
     )
 
-    if os.access(filepath, os.W_OK):
+    if os.access(filepath, os.W_OK) and file_size != float('inf'):
         # on Windows rename could fail if destination filepath exists
-        os.remove(filepath)
+        # we should simply choose a new name instead of brutal os.remove(filepath)
+        filepath = filepath + " (2)"
     os.rename(temp_filepath, filepath)
 
 

From 57cf717bd5be00c23cce50a136253577bd66763b Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Apr 2025 14:55:05 +0200
Subject: [PATCH 1222/1225] python-package.yml: disable the new flake8 F824
 check

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 70fcc335e6..17198de58a 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -31,7 +31,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82,F824 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with unittest

From c7e7525c9aec969984ac487982ba8f1876f68fd0 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Apr 2025 15:02:18 +0200
Subject: [PATCH 1223/1225] python-package.yml: disable the new flake8 F824
 check

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index 17198de58a..fc24cda3d6 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -31,7 +31,7 @@ jobs:
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82,F824 --show-source --statistics
+        flake8 . --count --select=E9,F63,F7,F82 --ignore=F824 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with unittest

From 1af8b714cdfed08523895dbb67de9bce631cf94e Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Apr 2025 16:05:03 +0200
Subject: [PATCH 1224/1225] python-package.yml: remove python 3.7 (no longer
 available in Ubuntu 24.04)

---
 .github/workflows/python-package.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
index fc24cda3d6..be2e88842c 100644
--- a/.github/workflows/python-package.yml
+++ b/.github/workflows/python-package.yml
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.8, pypy-3.9, pypy-3.10]
+        python-version: [3.8, 3.9, '3.10', '3.11', '3.12', '3.13', pypy-3.8, pypy-3.9, pypy-3.10]
 
     steps:
     - uses: actions/checkout@v4

From 049548f3f3f35e67ba8d3181c71fdc71d11cf260 Mon Sep 17 00:00:00 2001
From: Mort Yao <soi@mort.ninja>
Date: Sun, 27 Apr 2025 17:32:00 +0200
Subject: [PATCH 1225/1225] README.md: add --force-reinstall to pip because it
 is now necessary for upgrading from a VCS URL to work when the package
 version is unchanged (https://github.com/pypa/pip/issues/9397,
 https://github.com/pypa/pip/issues/5780)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a3cb7cea84..178ae0ac74 100644
--- a/README.md
+++ b/README.md
@@ -156,7 +156,7 @@ $ you-get https://github.com/soimort/you-get/archive/master.zip
 In order to get the latest ```develop``` branch without messing up the PIP, you can try:
 
 ```
-$ pip install --upgrade git+https://github.com/soimort/you-get@develop
+$ pip install --upgrade --force-reinstall git+https://github.com/soimort/you-get@develop
 ```
 
 ## Getting Started