Skip to content

Commit e3611e1

Browse files
fezcodefezcode
committed
build(ssg): discover dynamic routes + tune prerender crawler
Enumerate blog posts, series episodes, log entries, and story book pages so the crawler prerenders all ~460 routes instead of only the 140 static ones. Tighten the crawler: 8-way concurrency, domcontentloaded, 5s selector wait, 250ms settle, block images/fonts /media at the request layer.
1 parent 7580505 commit e3611e1

3 files changed

Lines changed: 104 additions & 11 deletions

File tree

‎pages/+onBeforePrerenderStart.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import { staticRoutes } from './routes.js';
1+
import { discoverAllRoutes } from './discoverRoutes.js';
22

33
export default function onBeforePrerenderStart() {
4-
return staticRoutes;
4+
return discoverAllRoutes();
55
}

‎pages/discoverRoutes.js‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import { readFileSync, readdirSync, existsSync } from 'node:fs';
2+
import { join } from 'node:path';
3+
import { staticRoutes } from './routes.js';
4+
5+
const PUBLIC = 'public';
6+
7+
function readJson(path) {
8+
try {
9+
return JSON.parse(readFileSync(path, 'utf8'));
10+
} catch {
11+
return null;
12+
}
13+
}
14+
15+
function blogRoutes() {
16+
const posts = readJson(join(PUBLIC, 'posts', 'posts.json')) || [];
17+
const out = new Set();
18+
for (const entry of posts) {
19+
if (!entry?.slug) continue;
20+
if (entry.series?.posts?.length) {
21+
out.add(`/blog/series/${entry.slug}`);
22+
for (const ep of entry.series.posts) {
23+
if (ep?.slug) out.add(`/blog/series/${entry.slug}/${ep.slug}`);
24+
}
25+
} else {
26+
out.add(`/blog/${entry.slug}`);
27+
}
28+
}
29+
return out;
30+
}
31+
32+
function logsRoutes() {
33+
const out = new Set();
34+
const root = join(PUBLIC, 'logs');
35+
if (!existsSync(root)) return out;
36+
for (const category of readdirSync(root, { withFileTypes: true })) {
37+
if (!category.isDirectory()) continue;
38+
const dir = join(root, category.name);
39+
for (const file of readdirSync(dir)) {
40+
if (!file.endsWith('.txt')) continue;
41+
const slug = file.slice(0, -4);
42+
out.add(`/logs/${category.name}/${slug}`);
43+
}
44+
}
45+
return out;
46+
}
47+
48+
function storyBookRoutes() {
49+
const out = new Set();
50+
const files = ['books_en.piml', 'books_tr.piml']
51+
.map((f) => join(PUBLIC, 'stories', f))
52+
.filter(existsSync);
53+
const bookIds = new Set();
54+
const bookEpisodes = new Map();
55+
for (const f of files) {
56+
const text = readFileSync(f, 'utf8');
57+
let currentBook = null;
58+
for (const line of text.split(/\r?\n/)) {
59+
const bookMatch = line.match(/\(bookId\)\s*(\S+)/);
60+
if (bookMatch) {
61+
currentBook = bookMatch[1];
62+
bookIds.add(currentBook);
63+
if (!bookEpisodes.has(currentBook)) bookEpisodes.set(currentBook, new Set());
64+
continue;
65+
}
66+
const epMatch = line.match(/^\s*\(id\)\s*(\S+)/);
67+
if (epMatch && currentBook) {
68+
bookEpisodes.get(currentBook).add(epMatch[1]);
69+
}
70+
}
71+
}
72+
for (const id of bookIds) out.add(`/stories/books/${id}`);
73+
for (const [book, eps] of bookEpisodes) {
74+
for (const ep of eps) out.add(`/stories/books/${book}/pages/${ep}`);
75+
}
76+
return out;
77+
}
78+
79+
export function discoverAllRoutes() {
80+
const all = new Set(staticRoutes);
81+
for (const r of blogRoutes()) all.add(r);
82+
for (const r of logsRoutes()) all.add(r);
83+
for (const r of storyBookRoutes()) all.add(r);
84+
return Array.from(all);
85+
}

‎scripts/prerender-crawl.mjs‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ import { existsSync } from 'node:fs';
33
import { join } from 'node:path';
44
import { preview } from 'vite';
55
import puppeteer from 'puppeteer';
6-
import { staticRoutes } from '../pages/routes.js';
6+
import { discoverAllRoutes } from '../pages/discoverRoutes.js';
7+
8+
const allRoutes = discoverAllRoutes();
79

810
const DIST = 'dist/client';
9-
const CONCURRENCY = 4;
10-
const PAGE_TIMEOUT = 45000;
11-
const RENDER_SETTLE_MS = 800;
11+
const CONCURRENCY = Number(process.env.PRERENDER_CONCURRENCY) || 8;
12+
const PAGE_TIMEOUT = 20000;
13+
const RENDER_SETTLE_MS = 250;
1214

1315
function routeToFile(route) {
1416
if (route === '/') return join(DIST, 'index.html');
@@ -26,11 +28,17 @@ async function crawlOne(browser, baseUrl, route) {
2628
page.on('console', (m) => {
2729
if (m.type() === 'error') consoleErrors.push(m.text());
2830
});
29-
await page.goto(target, { waitUntil: 'networkidle0', timeout: PAGE_TIMEOUT });
31+
await page.setRequestInterception(true);
32+
page.on('request', (req) => {
33+
const t = req.resourceType();
34+
if (t === 'image' || t === 'font' || t === 'media') return req.abort();
35+
req.continue();
36+
});
37+
await page.goto(target, { waitUntil: 'domcontentloaded', timeout: PAGE_TIMEOUT });
3038
try {
31-
await page.waitForSelector('#react-root > *', { timeout: 10000 });
39+
await page.waitForSelector('#react-root > *', { timeout: 5000 });
3240
} catch {
33-
return { route, skipped: 'no children rendered within 10s' };
41+
return { route, skipped: 'no children rendered within 5s' };
3442
}
3543
await new Promise((r) => setTimeout(r, RENDER_SETTLE_MS));
3644

@@ -59,7 +67,7 @@ async function crawlOne(browser, baseUrl, route) {
5967
}
6068

6169
async function main() {
62-
console.log(`prerender-crawl: ${staticRoutes.length} routes, concurrency ${CONCURRENCY}`);
70+
console.log(`prerender-crawl: ${allRoutes.length} routes, concurrency ${CONCURRENCY}`);
6371

6472
const server = await preview({
6573
preview: { port: 4287, strictPort: true, host: '127.0.0.1' },
@@ -72,7 +80,7 @@ async function main() {
7280
});
7381

7482
const results = [];
75-
const queue = [...staticRoutes];
83+
const queue = [...allRoutes];
7684
const workers = Array.from({ length: CONCURRENCY }, async () => {
7785
while (queue.length) {
7886
const route = queue.shift();

0 commit comments

Comments
 (0)