Update

fireattack · fireattack · commit 8faf6ce7d3cd · 2020-08-24T04:18:02.000-05:00
diff --git a/.gitignore b/.gitignore
@@ -123,3 +123,5 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+_test.py
+/.vscode
diff --git a/scraper_ameblo_api.py b/scraper_ameblo_api.py
@@ -65,9 +65,17 @@ def parse_list_new(blog_id, start_entry, until=None):
 def download_all(blog_id, save_folder='.', executor=None, until=None, last_entry='auto'):
 
     if last_entry == 'auto':
-        soup = get(f'https://ameblo.jp/{blog_id}/')
-        last_entry = re.search(r'entry-(\d+).html',
-                               soup.select_one(f'a[href*="{blog_id}/entry-"]')['href'])[1]
+        soup = get(f'https://ameblo.jp/{blog_id}/')        
+        if anchor := soup.select_one(f'a[href*="{blog_id}/entry-"]'):
+            last_entry = re.search(r'entry-(\d+).html', anchor['href'])[1]
+        else:
+            for s in soup('script'):
+                if s.text.startswith('window.INIT_DATA'):
+                    if m := re.search(r'entryMap.+?"(\d+)"', s.text):
+                        last_entry = m[1]
+                        break
+        if last_entry == 'auto':
+            raise Exception(f'[Error] cannot detect last entry for blog {blog_id}!')
         print(f'Info: {blog_id}\'s newest entry is {last_entry}.')
     if last_entry == until:
         print('No new update.')
diff --git a/util.py b/util.py
@@ -36,8 +36,10 @@ def get(url, headers=None, cookies=None, encoding='utf-8'):
     return BeautifulSoup(r.text, 'lxml')
 
 
-def download(url, filename=None, save_path='.', cookies=None, dry_run=False, dupe='skip',referer=None):
-    
+def download(url, filename=None, save_path='.', cookies=None, dry_run=False, dupe='skip',referer=None, placeholder=True):   
+    if dupe not in ['skip', 'overwrite', 'rename']:
+        raise ValueError('[Error] Invalid dupe method: {dupe} (must be either skip, overwrite or rename).')        
+
     def check_dupe(filename):
         if not filename.exists():
             return filename        
@@ -55,6 +57,7 @@ def check_dupe(filename):
                 i = i + 1
             print(f'[Warning] File already exists! Rename to {filename.name}.')
             return filename
+        
 
     if dry_run:
         return
@@ -88,10 +91,12 @@ def check_dupe(filename):
                 for chunk in r.iter_content(chunk_size=8192):
                     if chunk:       
                         fio.write(chunk)
+            return 200
         else:
             print(f'[Error] Get HTTP {r.status_code} from {url}.')
-            # An empty file will still be created. This is by design.
-            f.with_suffix(f.suffix + '.broken').open('wb').close()
+            if placeholder:
+                f.with_suffix(f.suffix + '.broken').open('wb').close()
+            return 404
                 
                 
 
