Skip to content

Commit ea29133

Browse files
N2D4N2D4
committed
Generation scripts now acquire a lock
1 parent da79285 commit ea29133

3 files changed

Lines changed: 131 additions & 100 deletions

File tree

‎scripts/generate-docs.ts‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import fs from "fs";
22
import path from "path";
33
import yaml from "yaml";
4-
import { PLATFORMS, copyFromSrcToDest, processMacros, writeFileSyncIfChanged } from "./utils";
4+
import { PLATFORMS, copyFromSrcToDest, processMacros, withGeneratorLock, writeFileSyncIfChanged } from "./utils";
55

66
interface DocObject {
77
platform?: string;
@@ -65,32 +65,36 @@ function processDocObject(obj: any, platforms: string[]): { result: any, validPa
6565
}
6666
}
6767

68-
const docsDir = path.resolve(__dirname, "..", "docs", "fern");
69-
const templateDir = path.join(docsDir, "docs", "pages-template");
70-
const ymlTemplatePath = path.join(docsDir, "docs-template.yml");
7168

72-
for (const platform of ["next", "js", "react", "python"]) {
73-
const destDir = path.join(docsDir, 'docs', `pages-${platform}`);
7469

75-
const mainYmlContent = fs.readFileSync(ymlTemplatePath, "utf-8");
76-
const macroProcessed = processMacros(mainYmlContent, PLATFORMS[platform]);
77-
const template = yaml.parse(macroProcessed);
78-
const { result: processed, validPaths: processedValidPaths } = processDocObject(template, PLATFORMS[platform]);
79-
const output = yaml.stringify(processed);
80-
writeFileSyncIfChanged(path.join(docsDir, `${platform}.yml`), output);
70+
withGeneratorLock(async () => {
71+
const docsDir = path.resolve(__dirname, "..", "docs", "fern");
72+
const templateDir = path.join(docsDir, "docs", "pages-template");
73+
const ymlTemplatePath = path.join(docsDir, "docs-template.yml");
8174

82-
// Copy the entire template directory, processing macros for each file
83-
copyFromSrcToDest({
84-
srcDir: templateDir,
85-
destDir,
86-
editFn: (relativePath, content) => {
87-
return processMacros(content, PLATFORMS[platform]);
88-
},
89-
filterFn: (relativePath) => {
90-
if (relativePath.endsWith('.mdx') && !relativePath.startsWith('snippets')) {
91-
return processedValidPaths.includes(relativePath);
75+
for (const platform of ["next", "js", "react", "python"]) {
76+
const destDir = path.join(docsDir, 'docs', `pages-${platform}`);
77+
78+
const mainYmlContent = fs.readFileSync(ymlTemplatePath, "utf-8");
79+
const macroProcessed = processMacros(mainYmlContent, PLATFORMS[platform]);
80+
const template = yaml.parse(macroProcessed);
81+
const { result: processed, validPaths: processedValidPaths } = processDocObject(template, PLATFORMS[platform]);
82+
const output = yaml.stringify(processed);
83+
writeFileSyncIfChanged(path.join(docsDir, `${platform}.yml`), output);
84+
85+
// Copy the entire template directory, processing macros for each file
86+
copyFromSrcToDest({
87+
srcDir: templateDir,
88+
destDir,
89+
editFn: (relativePath, content) => {
90+
return processMacros(content, PLATFORMS[platform]);
91+
},
92+
filterFn: (relativePath) => {
93+
if (relativePath.endsWith('.mdx') && !relativePath.startsWith('snippets')) {
94+
return processedValidPaths.includes(relativePath);
95+
}
96+
return true;
9297
}
93-
return true;
94-
}
95-
});
96-
}
98+
});
99+
}
100+
}).catch(console.error);

‎scripts/generate-sdks.ts‎

Lines changed: 65 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import fs from "fs";
22
import path from "path";
3-
import { COMMENT_BLOCK, COMMENT_LINE, PLATFORMS, copyFromSrcToDest, processMacros, writeFileSyncIfChanged } from "./utils";
3+
import { COMMENT_BLOCK, COMMENT_LINE, PLATFORMS, copyFromSrcToDest, processMacros, withGeneratorLock, writeFileSyncIfChanged } from "./utils";
44

55
/**
66
* Main function to generate from a template:
@@ -123,65 +123,67 @@ function baseEditFn(options: {
123123
}
124124

125125

126-
const baseDir = path.resolve(__dirname, "..", "packages");
127-
const srcDir = path.resolve(baseDir, "template");
128-
129-
// Copy package-template.json to package.json in the template,
130-
// applying macros and adding a comment field.
131-
const packageTemplateContent = fs.readFileSync(
132-
path.join(srcDir, "package-template.json"),
133-
"utf-8"
134-
);
135-
const processedPackageJson = processMacros(packageTemplateContent, PLATFORMS["template"]);
136-
writeFileSyncIfChanged(
137-
path.join(srcDir, "package.json"),
138-
processPackageJson(processedPackageJson)
139-
);
140-
141-
generateFromTemplate({
142-
src: srcDir,
143-
dest: path.resolve(baseDir, "js"),
144-
editFn: (relativePath, content) => {
145-
return baseEditFn({ relativePath, content, platforms: PLATFORMS["js"] });
146-
},
147-
filterFn: (relativePath) => {
148-
const ignores = [
149-
"postcss.config.js",
150-
"tailwind.config.js",
151-
"quetzal.config.json",
152-
"components.json",
153-
".env",
154-
".env.local",
155-
"scripts/",
156-
"quetzal-translations/",
157-
"src/components/",
158-
"src/components-page/",
159-
"src/generated/",
160-
"src/providers/",
161-
"src/global.css",
162-
"src/global.d.ts",
163-
];
164-
165-
if (ignores.some((ignorePath) => relativePath.startsWith(ignorePath)) || relativePath.endsWith(".tsx")) {
166-
return false;
167-
} else {
168-
return true;
169-
}
170-
},
171-
});
172-
173-
generateFromTemplate({
174-
src: srcDir,
175-
dest: path.resolve(baseDir, "stack"),
176-
editFn: (relativePath, content) => {
177-
return baseEditFn({ relativePath, content, platforms: PLATFORMS["next"] });
178-
},
179-
});
180-
181-
generateFromTemplate({
182-
src: srcDir,
183-
dest: path.resolve(baseDir, "react"),
184-
editFn: (relativePath, content) => {
185-
return baseEditFn({ relativePath, content, platforms: PLATFORMS["react"] });
186-
},
187-
});
126+
withGeneratorLock(async () => {
127+
const baseDir = path.resolve(__dirname, "..", "packages");
128+
const srcDir = path.resolve(baseDir, "template");
129+
130+
// Copy package-template.json to package.json in the template,
131+
// applying macros and adding a comment field.
132+
const packageTemplateContent = fs.readFileSync(
133+
path.join(srcDir, "package-template.json"),
134+
"utf-8"
135+
);
136+
const processedPackageJson = processMacros(packageTemplateContent, PLATFORMS["template"]);
137+
writeFileSyncIfChanged(
138+
path.join(srcDir, "package.json"),
139+
processPackageJson(processedPackageJson)
140+
);
141+
142+
generateFromTemplate({
143+
src: srcDir,
144+
dest: path.resolve(baseDir, "js"),
145+
editFn: (relativePath, content) => {
146+
return baseEditFn({ relativePath, content, platforms: PLATFORMS["js"] });
147+
},
148+
filterFn: (relativePath) => {
149+
const ignores = [
150+
"postcss.config.js",
151+
"tailwind.config.js",
152+
"quetzal.config.json",
153+
"components.json",
154+
".env",
155+
".env.local",
156+
"scripts/",
157+
"quetzal-translations/",
158+
"src/components/",
159+
"src/components-page/",
160+
"src/generated/",
161+
"src/providers/",
162+
"src/global.css",
163+
"src/global.d.ts",
164+
];
165+
166+
if (ignores.some((ignorePath) => relativePath.startsWith(ignorePath)) || relativePath.endsWith(".tsx")) {
167+
return false;
168+
} else {
169+
return true;
170+
}
171+
},
172+
});
173+
174+
generateFromTemplate({
175+
src: srcDir,
176+
dest: path.resolve(baseDir, "stack"),
177+
editFn: (relativePath, content) => {
178+
return baseEditFn({ relativePath, content, platforms: PLATFORMS["next"] });
179+
},
180+
});
181+
182+
generateFromTemplate({
183+
src: srcDir,
184+
dest: path.resolve(baseDir, "react"),
185+
editFn: (relativePath, content) => {
186+
return baseEditFn({ relativePath, content, platforms: PLATFORMS["react"] });
187+
},
188+
});
189+
}).catch(console.error);

‎scripts/utils.ts‎

Lines changed: 36 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,36 @@ export const PLATFORMS = {
1313
"python": ['python', 'python-like'],
1414
}
1515

16+
export const withGeneratorLock = async <T>(fn: () => Promise<T>) => {
17+
const lockFilePath = path.resolve(__dirname, "../generator-lock-file.untracked.lock");
18+
while (true) {
19+
try {
20+
fs.writeFileSync(lockFilePath, Date.now().toString(), { flag: 'wx' });
21+
break;
22+
} catch (e) {
23+
if ("code" in e && e.code === "EEXIST") {
24+
const millis = +fs.readFileSync(lockFilePath, 'utf-8');
25+
if (Date.now() - millis > 5 * 60 * 1000) {
26+
console.warn(`Generator lock file ${lockFilePath} exists, but is older than 5 minutes. Assuming it's stale and deleting.`);
27+
fs.unlinkSync(lockFilePath); // TODO: this should be done atomically
28+
await new Promise((resolve) => setTimeout(resolve, 5000));
29+
continue;
30+
} else {
31+
console.log(`Generator lock file ${lockFilePath} exists. Waiting for it to be released...`);
32+
await new Promise((resolve) => setTimeout(resolve, 2000 * Math.random()));
33+
continue;
34+
}
35+
} else {
36+
throw e;
37+
}
38+
}
39+
}
40+
try {
41+
return await fn();
42+
} finally {
43+
fs.unlinkSync(lockFilePath);
44+
}
45+
}
1646

1747
export function processMacros(content: string, platforms: string[]): string {
1848
const lines = content.split('\n');
@@ -242,18 +272,13 @@ export function processMacros(content: string, platforms: string[]): string {
242272
}
243273

244274
export function writeFileSyncIfChanged(path: string, content: string | Buffer): void {
275+
if (typeof content === 'string') {
276+
content = Buffer.from(content);
277+
}
245278
if (fs.existsSync(path)) {
246-
const existingContent = fs.readFileSync(path);
247-
if (Buffer.isBuffer(content)) {
248-
// For binary files, compare buffers
249-
if (Buffer.compare(existingContent, content) === 0) {
250-
return;
251-
}
252-
} else {
253-
// For text files, compare strings
254-
if (existingContent.toString('utf-8') === content) {
255-
return;
256-
}
279+
const existingContent = fs.readFileSync(path, { encoding: null });
280+
if (Buffer.compare(existingContent, content) === 0) {
281+
return;
257282
}
258283
}
259284
fs.writeFileSync(path, content);

0 commit comments

Comments
 (0)