scraper_fantia: add filename/dir templating

fireattack · fireattack · commit 88bfa2b627b8 · 2026-01-13T00:00:09.000+08:00
diff --git a/scraper_fantia.py b/scraper_fantia.py
@@ -1,6 +1,7 @@
 import concurrent.futures
 import re
 import webbrowser
+from datetime import datetime
 from pathlib import Path
 from urllib.parse import urljoin
 
@@ -14,17 +15,27 @@
 HTML_POSTLIST = "https://fantia.jp/fanclubs/{}/posts?page={}"
 DEFAULT_UA = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'
 
+# Default templates for directory and filename formatting
+# Directory template variables:
+#   {fanclub_id}, {fanclub_name}, {fanclub_full_name}, {creator_name}
+#   {post_id}, {post_title}, {post_date}, {rating}
+# Filename template variables (in addition to directory ones):
+#   {content_id}, {idx}, {stem_cleaned}
+# Note: extension (.ext) is always appended automatically and not part of the template
+DEFAULT_DIR_TEMPLATE = '{fanclub_full_name} ({fanclub_id})/{post_id} {post_title}'
+DEFAULT_FILENAME_TEMPLATE = '{content_id}{idx} {stem_cleaned}'
+
+
 class FantiaDownloader:
-    def __init__(self, key, fanclub=None, output='.', subfolder_template=None, skip_existing=True, quick_stop=True):
+    def __init__(self, key, fanclub=None, output='.', dir_template=None, filename_template=None, skip_existing=True, quick_stop=True):
         super().__init__()
         self.key = key
         self.fanclub = fanclub
         if not self.fanclub:
             print('[W] no fanclub id is given. "downloadAll()" won\'t work.')
         self.output = Path(output)
-        self.subfolder_template = subfolder_template or '{fanclub_full_name} ({fanclub_id})'
-        self.subfolder_name = None
-        #TODO: self.filename_template = '{id} {title}'
+        self.dir_template = dir_template or DEFAULT_DIR_TEMPLATE
+        self.filename_template = filename_template or DEFAULT_FILENAME_TEMPLATE
         self.skip_existing = skip_existing
         self.quick_stop = skip_existing and quick_stop
         self.fanclub_info = None
@@ -56,13 +67,30 @@ def download_all(self):
             with self.fetch(API_FANCLUB.format(self.fanclub)) as r:
                 self.update_fanclub_info(r.json()['fanclub'])
 
-        output_full = self.output / self.subfolder_name
-        print(f'Fanclub {self.fanclub}: download all to {output_full}.')
-        if output_full.exists():
-            existing_ids = [m[1] for f in output_full.iterdir() if (m := re.match(r'(\d+) *', f.name))]
+        # Get base output dir (fanclub level) for checking existing downloads
+        # Extract only the fanclub-level part of the template (before any post-level placeholders)
+        fanclub_subs = self._get_fanclub_substitutes()
+        # Split dir_template and only format the parts that don't require post info
+        template_parts = self.dir_template.split('/')
+        base_parts = []
+        for part in template_parts:
+            if any(p in part for p in ['{post_id}', '{post_title}', '{post_date}', '{rating}']):
+                break
+            base_parts.append(part.format(**fanclub_subs))
+        base_dir = self.output / '/'.join(base_parts) if base_parts else self.output
+
+        print(f'Fanclub {self.fanclub}: download all to {base_dir}.')
+        existing_ids = []
+        if base_dir.exists():
+            # Check for subdirectories starting with post_id (new structure)
+            for f in base_dir.iterdir():
+                if f.is_dir() and (m := re.match(r'^(\d+)\b', f.name)):
+                    existing_ids.append(m.group(1))
+            # Also check for files starting with post_id (old flat structure)
+            for f in base_dir.iterdir():
+                if f.is_file() and (m := re.match(r'^(\d+)\b', f.name)):
+                    existing_ids.append(m.group(1))
             existing_ids = list(dict.fromkeys(existing_ids))
-        else:
-            existing_ids = []
         print(f'{len(existing_ids)} ID(s) have already been downloaded.')
 
         def fetch_all():
@@ -110,15 +138,53 @@ def fetch_gallery_page(self, page):
 
     def update_fanclub_info(self, d):
         self.fanclub_info = d
-        substitutes = dict(
-            fanclub_full_name=self.fanclub_info['fanclub_name_with_creator_name'],
-            fanclub_name=self.fanclub_info['fanclub_name'],
-            creater_name=self.fanclub_info['creator_name'],
+
+    def _get_fanclub_substitutes(self):
+        """Get template substitutes from fanclub info."""
+        if not self.fanclub_info:
+            return {}
+        return dict(
+            fanclub_full_name=safeify(str(self.fanclub_info['fanclub_name_with_creator_name'])),
+            fanclub_name=safeify(str(self.fanclub_info['fanclub_name'])),
+            creator_name=safeify(str(self.fanclub_info['creator_name'])),
             fanclub_id=self.fanclub_info['id']
         )
-        substitutes = {k: safeify(str(v)) for k, v in substitutes.items()}
-        if not self.subfolder_name:
-            self.subfolder_name = self.subfolder_template.format(**substitutes)
+
+    def _get_post_substitutes(self, post_data):
+        """Get template substitutes from post data."""
+        # Parse posted_at date (format: "Fri, 16 Aug 2024 07:15:36 +0900")
+        post_date = ''
+        if posted_at := post_data.get('posted_at'):
+            try:
+                dt = datetime.strptime(posted_at, "%a, %d %b %Y %H:%M:%S %z")
+                post_date = dt.strftime('%Y%m%d')
+            except ValueError:
+                post_date = ''
+
+        return dict(
+            post_id=post_data.get('id', ''),
+            post_title=safeify(str(post_data.get('title', ''))),
+            post_date=post_date,
+            rating=post_data.get('rating', '')
+        )
+
+    def _format_dir(self, post_substitutes):
+        """Format directory path using template and substitutes."""
+        subs = {**self._get_fanclub_substitutes(), **post_substitutes}
+        return self.dir_template.format(**subs)
+
+    def _format_filename(self, post_substitutes, content_id, idx_string, stem_cleaned, ext):
+        """Format filename using template and substitutes."""
+        subs = {
+            **self._get_fanclub_substitutes(),
+            **post_substitutes,
+            'content_id': content_id,
+            'idx': idx_string,
+            'stem_cleaned': stem_cleaned
+        }
+        # Format the template, strip to remove trailing spaces, then append extension
+        filename_without_ext = self.filename_template.format(**subs).strip()
+        return f'{filename_without_ext}.{ext}'
 
     def get_post_photos(self, id):
         print(f'Fetching post {id}...')
@@ -139,27 +205,37 @@ def get_post_photos(self, id):
         if not self.fanclub_info:
             self.update_fanclub_info(d['post']['fanclub'])
 
+        post_data = d['post']
+        post_subs = self._get_post_substitutes(post_data)
+        output_dir = self.output / self._format_dir(post_subs)
+        # Ensure multi-level directory exists
+        output_dir.mkdir(parents=True, exist_ok=True)
+
         with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
-            if thumb := d['post'].get('thumb', None):
+            if thumb := post_data.get('thumb', None):
                 img_url = thumb['original']
                 stem, ext = get_webname(img_url).rsplit('.', 1)
-                ex.submit(download, img_url, filename= self.output / self.subfolder_name / f'{id} !cover.{ext}', verbose=1)
-            if post_contents := d['post'].get('post_contents', None):
+                cover_filename = f'!cover.{ext}'
+                ex.submit(download, img_url, filename=output_dir / cover_filename, verbose=1)
+            if post_contents := post_data.get('post_contents', None):
                 for c in post_contents:
                     cid = c['id']
                     if photos := c.get('post_content_photos', None):
                         for idx, p in enumerate(photos, 1):
                             img_url = p['url']['original']
                             stem, ext = get_webname(img_url).rsplit('.', 1)
                             # Clean up the filename; remove all the UUID-ish garbage
-                            stem = re.sub(r'^[0-9a-fA-F]{8}_(.+)$', r'\1', stem)
-                            stem = re.sub(r'[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', '', stem)
+                            stem_cleaned = re.sub(r'^[0-9a-fA-F]{8}_(.+)$', r'\1', stem)
+                            stem_cleaned = re.sub(r'[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', '', stem_cleaned)
+                            stem_cleaned = stem_cleaned.strip()
                             idx_string = '_' + str(idx).zfill(len(str(len(photos)))) if len(photos) > 1 else ''
-                            stem = f'{id} {cid}{idx_string} {stem}'.strip()
-                            ex.submit(download, img_url, filename=self.output / self.subfolder_name / f'{stem}.{ext}', verbose=1)
+                            filename = self._format_filename(post_subs, cid, idx_string, stem_cleaned, ext)
+                            ex.submit(download, img_url, filename=output_dir / filename, verbose=1)
                     if 'download_uri' in c:
                         dl_url = urljoin('https://fantia.jp', c['download_uri'])
-                        ex.submit(download, dl_url, filename=self.output / self.subfolder_name / f'{id} {cid} {c["filename"]}', session=self.session, verbose=1)
+                        # For download files, use original filename with content_id prefix
+                        dl_filename = f'{cid} {c["filename"]}'
+                        ex.submit(download, dl_url, filename=output_dir / dl_filename, session=self.session, verbose=1)
 
 if __name__ == "__main__":
     pass
diff --git a/util.py b/util.py
@@ -1039,7 +1039,9 @@ def replace_suffix(f, content_type):
                 # don't replace equivalent extensions
                 ext_alias_groups = [
                     ['.mp4', '.m4a', '.m4v', '.m4s'],
-                    ['.jpg', '.jpeg']
+                    ['.jpg', '.jpeg'],
+                    ['.m3u', '.m3u8'],
+                    ['.mp2t', '.ts'],
                 ]
                 for aliases in ext_alias_groups:
                     if f.suffix.lower() in aliases and header_suffix in aliases:
