Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions packages/rstack/src/fmt/cacheIdentity.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { hash } from 'node:crypto';
import { hash as createDigest } from 'node:crypto';
import { isAbsolute } from 'node:path';
import stableStringify from 'fast-json-stable-stringify';
import { fmtCacheVersion } from './cacheStore.ts';
Expand All @@ -12,7 +12,9 @@ type CacheKeyResolver = (filePath: string) => string | undefined;
type OptionsHasher = (options: ResolvedFmtOptions) => string | undefined;
type PluginFingerprints = ReadonlyMap<string, string>;

const sha256 = (content: string | Uint8Array): string => hash('sha256', content, 'hex');
const cacheHashLength = 16;
const createCacheHash = (content: string | Uint8Array): string =>
createDigest('sha256', content, 'base64url').slice(0, cacheHashLength);

/** Identifies formatter behavior shared by all cache entries in this process. */
const cacheNamespace: string = JSON.stringify([fmtCacheVersion, RSTACK_VERSION, PRETTIER_VERSION]);
Expand Down Expand Up @@ -55,7 +57,7 @@ const createOptionsHasher = (pluginFingerprints?: PluginFingerprints): OptionsHa
}
value = { ...options, plugins: fingerprints };
}
hash = sha256(stableStringify(value));
hash = createCacheHash(stableStringify(value));
} catch {
// Circular or unreadable options cannot be cached.
}
Expand All @@ -65,4 +67,10 @@ const createOptionsHasher = (pluginFingerprints?: PluginFingerprints): OptionsHa
};
};

export { cacheNamespace, createCacheKeyResolver, createOptionsHasher, sha256 };
export {
cacheHashLength,
cacheNamespace,
createCacheHash,
createCacheKeyResolver,
createOptionsHasher,
};
226 changes: 162 additions & 64 deletions packages/rstack/src/fmt/cacheStore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,40 @@ import { randomUUID } from 'node:crypto';
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
import path from 'node:path';

const fmtCacheFileName = 'v1.json';
const fmtCacheVersion = 1;
const fmtCacheFileName = 'cache.json';
const fmtCacheVersion = 2;

type FmtCacheState = 'clean' | 'dirty' | 'unsupported';
type FmtCacheEntry =
| readonly [contentHash: string, optionsHash: string, state: 'clean' | 'dirty']
| readonly [contentHash: string | null, optionsHash: string, state: 'unsupported'];
const fileEntryWidth = 4;
const contentHashOffset = 1;
const optionsIndexOffset = 2;
const stateOffset = 3;

const fmtCacheStates = ['clean', 'dirty', 'unsupported'] as const;
type FmtCacheState = (typeof fmtCacheStates)[number];
type FmtCacheStateId = 0 | 1 | 2;

const fmtCacheStateIds = {
clean: 0,
dirty: 1,
unsupported: 2,
} as const satisfies Record<FmtCacheState, FmtCacheStateId>;

type FmtCacheFileValue = string | number;
type FmtCacheEntry = readonly [contentHash: string, optionsHash: string, state: FmtCacheState];

interface FmtCacheFile {
version: typeof fmtCacheVersion;
namespace: string;
files: Record<string, FmtCacheEntry>;
options: string[];
/** Repeated tuples of file path, content hash, options index, and numeric state. */
files: FmtCacheFileValue[];
}

interface ParsedFmtCacheFile {
cache: FmtCacheFile;
fileOffsets: Map<string, number>;
optionsIndexes: Map<string, number>;
optionsUseCounts: number[];
}

interface FmtCacheStore {
Expand All @@ -23,66 +45,61 @@ interface FmtCacheStore {
save(): Promise<boolean>;
}

const createEmptyCache = (namespace: string): FmtCacheFile => ({
version: fmtCacheVersion,
namespace,
files: Object.create(null) as Record<string, FmtCacheEntry>,
const createEmptyCache = (namespace: string): ParsedFmtCacheFile => ({
cache: {
version: fmtCacheVersion,
namespace,
options: [],
files: [],
},
fileOffsets: new Map(),
optionsIndexes: new Map(),
optionsUseCounts: [],
});

const parseCacheEntry = (value: unknown): FmtCacheEntry | undefined => {
if (!Array.isArray(value) || value.length !== 3 || typeof value[1] !== 'string') {
return;
}

if (value[2] === 'unsupported') {
return value[0] === null || typeof value[0] === 'string'
? [value[0], value[1], value[2]]
: undefined;
}
if (typeof value[0] !== 'string' || (value[2] !== 'clean' && value[2] !== 'dirty')) {
return;
}

return [value[0], value[1], value[2]];
};

const parseCacheFile = (content: string): FmtCacheFile | undefined => {
const parseCacheFile = (content: string): ParsedFmtCacheFile | undefined => {
let value: unknown;
try {
value = JSON.parse(content);
} catch {
return;
}

if (typeof value !== 'object' || value === null || Array.isArray(value)) {
return;
}

const cache = value as FmtCacheFile;
const { version, namespace, options, files } = cache;
if (
typeof value !== 'object' ||
value === null ||
Array.isArray(value) ||
!('version' in value) ||
value.version !== fmtCacheVersion ||
!('namespace' in value) ||
typeof value.namespace !== 'string' ||
!('files' in value) ||
typeof value.files !== 'object' ||
value.files === null ||
Array.isArray(value.files)
version !== fmtCacheVersion ||
typeof namespace !== 'string' ||
!Array.isArray(options) ||
!Array.isArray(files) ||
files.length % fileEntryWidth !== 0
) {
return;
}

const files = Object.create(null) as Record<string, FmtCacheEntry>;
for (const [filePath, rawEntry] of Object.entries(value.files)) {
const entry = parseCacheEntry(rawEntry);
if (!entry) {
return;
}
files[filePath] = entry;
const optionsIndexes = new Map<string, number>();
for (let index = 0; index < options.length; index++) {
optionsIndexes.set(options[index], index);
}

const fileOffsets = new Map<string, number>();
const optionsUseCounts = new Array<number>(options.length).fill(0);
for (let offset = 0; offset < files.length; offset += fileEntryWidth) {
const filePath = files[offset] as string;
const optionsIndex = files[offset + optionsIndexOffset] as number;
fileOffsets.set(filePath, offset);
optionsUseCounts[optionsIndex]++;
Comment thread
chenjiahan marked this conversation as resolved.
}

return {
version: fmtCacheVersion,
namespace: value.namespace,
files,
cache,
fileOffsets,
optionsIndexes,
optionsUseCounts,
};
};

Expand All @@ -100,43 +117,124 @@ const getTemporaryPath = (filePath: string): string =>
class FmtCacheStoreImpl implements FmtCacheStore {
readonly #filePath: string;
readonly #cache: FmtCacheFile;
readonly #fileOffsets: Map<string, number>;
readonly #optionsIndexes: Map<string, number>;
readonly #optionsUseCounts: number[];
#savedContent: string | undefined;
#changed: boolean;

constructor(
filePath: string,
cache: FmtCacheFile,
parsed: ParsedFmtCacheFile,
savedContent: string | undefined,
changed: boolean,
) {
this.#filePath = filePath;
this.#cache = cache;
this.#cache = parsed.cache;
this.#fileOffsets = parsed.fileOffsets;
this.#optionsIndexes = parsed.optionsIndexes;
this.#optionsUseCounts = parsed.optionsUseCounts;
this.#savedContent = savedContent;
this.#changed = changed;
}

get(filePath: string): FmtCacheEntry | undefined {
return this.#cache.files[filePath];
const offset = this.#fileOffsets.get(filePath);
if (offset === undefined) {
return;
}

const { files, options } = this.#cache;
const contentHash = files[offset + contentHashOffset] as string;
const optionsHash = options[files[offset + optionsIndexOffset] as number];
const state = fmtCacheStates[files[offset + stateOffset] as FmtCacheStateId];
return [contentHash, optionsHash, state];
}

set(filePath: string, entry: FmtCacheEntry): void {
const current = this.#cache.files[filePath];
if (current?.[0] === entry[0] && current[1] === entry[1] && current[2] === entry[2]) {
return;
const { files, options } = this.#cache;
const [contentHash, optionsHash, state] = entry;
const stateId = fmtCacheStateIds[state];
const offset = this.#fileOffsets.get(filePath);

if (offset !== undefined) {
const currentOptionsIndex = files[offset + optionsIndexOffset] as number;
if (
files[offset + contentHashOffset] === contentHash &&
options[currentOptionsIndex] === optionsHash &&
files[offset + stateOffset] === stateId
) {
return;
}

const optionsIndex = this.#getOrCreateOptionsIndex(optionsHash);
if (currentOptionsIndex !== optionsIndex) {
this.#optionsUseCounts[currentOptionsIndex]--;
this.#optionsUseCounts[optionsIndex]++;
files[offset + optionsIndexOffset] = optionsIndex;
}
files[offset + contentHashOffset] = contentHash;
files[offset + stateOffset] = stateId;
} else {
const optionsIndex = this.#getOrCreateOptionsIndex(optionsHash);
const nextOffset = files.length;
files.push(filePath, contentHash, optionsIndex, stateId);
this.#fileOffsets.set(filePath, nextOffset);
this.#optionsUseCounts[optionsIndex]++;
}

this.#cache.files[filePath] =
entry[2] === 'unsupported'
? [entry[0], entry[1], 'unsupported']
: [entry[0], entry[1], entry[2]];
this.#changed = true;
}

#getOrCreateOptionsIndex(optionsHash: string): number {
const current = this.#optionsIndexes.get(optionsHash);
if (current !== undefined) {
return current;
}

const index = this.#cache.options.length;
this.#cache.options.push(optionsHash);
this.#optionsIndexes.set(optionsHash, index);
this.#optionsUseCounts.push(0);
return index;
}

#compactUnusedOptions(): void {
if (!this.#optionsUseCounts.includes(0)) {
return;
}

const { files, options } = this.#cache;
const nextOptions: string[] = [];
const nextUseCounts: number[] = [];
const remappedIndexes = new Int32Array(options.length).fill(-1);
for (let index = 0; index < options.length; index++) {
const useCount = this.#optionsUseCounts[index];
if (useCount > 0) {
remappedIndexes[index] = nextOptions.length;
nextOptions.push(options[index]);
nextUseCounts.push(useCount);
}
}
for (let offset = 0; offset < files.length; offset += fileEntryWidth) {
const currentIndex = files[offset + optionsIndexOffset] as number;
files[offset + optionsIndexOffset] = remappedIndexes[currentIndex];
}

options.splice(0, options.length, ...nextOptions);
this.#optionsUseCounts.splice(0, this.#optionsUseCounts.length, ...nextUseCounts);
this.#optionsIndexes.clear();
for (let index = 0; index < options.length; index++) {
this.#optionsIndexes.set(options[index], index);
}
}

async save(): Promise<boolean> {
if (!this.#changed) {
return false;
}

this.#compactUnusedOptions();
const content = serializeCache(this.#cache);
if (content === this.#savedContent) {
this.#changed = false;
Expand Down Expand Up @@ -164,13 +262,13 @@ const loadFmtCacheStore = async (filePath: string, namespace: string): Promise<F

try {
const content = await readFile(filePath, 'utf8');
const cache = parseCacheFile(content);
if (!cache) {
const parsed = parseCacheFile(content);
if (!parsed) {
return new FmtCacheStoreImpl(filePath, emptyCache, undefined, true);
}

return cache.namespace === namespace
? new FmtCacheStoreImpl(filePath, cache, content, false)
return parsed.cache.namespace === namespace
? new FmtCacheStoreImpl(filePath, parsed, content, false)
: new FmtCacheStoreImpl(filePath, emptyCache, undefined, true);
} catch (error) {
const missing = isFileNotFoundError(error);
Expand Down
2 changes: 1 addition & 1 deletion packages/rstack/src/fmt/runner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ const isCachedUnsupported = ({ file, cache }: FmtFileRunTask): boolean => {
return false;
}
return (
cache.entry[0] === null &&
cache.entry[0] === '' &&
cache.entry[1] === cache.optionsHash &&
cache.entry[2] === 'unsupported' &&
hasDottedBasename(file.path)
Expand Down
7 changes: 4 additions & 3 deletions packages/rstack/src/fmt/worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ interface FormatFileTask {
cache?: FmtFileCache;
}

const hashContent = (content: string | Uint8Array): string => hash('sha256', content, 'hex');
const hashContent = (content: string | Uint8Array): string =>
hash('sha256', content, 'base64url').slice(0, 16);

/**
* Use synchronous direct I/O inside the dedicated worker to avoid libuv
Expand Down Expand Up @@ -44,7 +45,7 @@ const formatFile = async ({
if (cache?.entry && cache.entry[1] === cache.optionsHash) {
const { entry } = cache;
if (entry[2] === 'unsupported') {
if (entry[0] === null) {
if (entry[0] === '') {
if (hasDottedBasename(file.path)) {
return { status: 'unsupported' };
}
Expand Down Expand Up @@ -72,7 +73,7 @@ const formatFile = async ({
status: 'unsupported',
cacheEntry: [
hasDottedBasename(file.path)
? null
? ''
: (contentHash ?? hashContent(sourceBuffer ?? readFileSync(file.path))),
cache.optionsHash,
'unsupported',
Expand Down
Loading