diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36664002..34c85f22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,14 +12,14 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x, 16.x, 18.x] + node-version: [18.x, 20.x, 22.x, 24.x] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: submodules: recursive - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v3 + uses: actions/setup-node@v6 with: node-version: ${{ matrix.node-version }} - run: npm i diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 479cf9a8..0cfe84d7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,19 +3,25 @@ on: push: branches: - master + +permissions: + id-token: write # Required for OIDC / npm trusted publishing + contents: write # To create releases + jobs: release: name: Release runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: fetch-depth: 0 - name: Setup Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v6 with: - node-version: 14 + node-version: 24 + registry-url: 'https://registry.npmjs.org' - name: Install dependencies run: npm i - name: Build module @@ -23,5 +29,4 @@ jobs: - name: Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} run: npx semantic-release diff --git a/.gitignore b/.gitignore index 5c11bde6..5d115cf3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ coverage npm-debug.log .nyc_output lib +.vscode diff --git a/LICENSE b/LICENSE index a0396333..18ed0788 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2022 Dmitry Shirokov +Copyright (c) 2026 Dmitry Shirokov Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/README.md b/README.md index 1e85d46c..d22f20bb 100644 --- a/README.md +++ b/README.md @@ -65,9 +65,9 @@ _maxmind.open(filepath, [options])_ - `filepath`: `` Path to the binary mmdb database file. - `options`: `` - `cache`: `` Cache options. Under the bonnet module uses [tiny-lru](https://github.com/avoidwork/tiny-lru) cache. - - `max`: `` Max cache items to keep in memory. _Default_: `6000`. + - `max`: `` Max cache items to keep in memory. _Default_: `10_000`. - `watchForUpdates`: `` Supports reloading the reader when changes occur to the database that is loaded. _Default_: `false`. - - `watchForUpdatesNonPersistent`: `` Controlls wether the watcher should be persistent or not. If it is persistent, a node process will be blocked in watching state if the watcher is the only thing still running in the program. _Default_: `false`. + - `watchForUpdatesNonPersistent`: `` Controls whether the watcher should be persistent or not. If it is persistent, a node process will be blocked in watching state if the watcher is the only thing still running in the program. _Default_: `false`. - `watchForUpdatesHook`: `` Hook function that is fired on database update. _Default_: `null`. ## Does it work in browser? diff --git a/jest.config.js b/jest.config.js index c9bbccc2..722ea2db 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,7 +1,7 @@ module.exports = { testEnvironment: 'node', testRegex: '.*test.ts$', - transform: { '^.+\\.ts?$': 'ts-jest' }, + transform: { '^.+\\.ts?$': '@swc/jest' }, moduleFileExtensions: ['ts', 'js', 'json'], rootDir: 'src', collectCoverage: true, diff --git a/package.json b/package.json index fd209b0c..e07b8b5a 100644 --- a/package.json +++ b/package.json @@ -23,31 +23,30 @@ "Thomas Birke @quafzi ", "Afzaal Ameer @afzaalace", "Andrew N Golovkov @AndorCS", - "Gregory Oschwald @oschwald" + "Gregory Oschwald @oschwald", + "Mariano Facundo Scigliano @MarianoFacundoArch" ], "dependencies": { - "mmdb-lib": "2.0.2", - "tiny-lru": "8.0.2" + "mmdb-lib": "3.0.2", + "tiny-lru": "13.0.0" }, "devDependencies": { - "@types/ip6addr": "0.2.3", - "@types/jest": "28.1.8", - "@types/netmask": "1.0.30", - "@types/node": "16.11.60", - "@types/sinon": "10.0.13", - "ip-address": "8.1.0", + "@swc/core": "1.15.40", + "@swc/jest": "0.2.39", + "@types/ip6addr": "0.2.6", + "@types/jest": "30.0.0", + "@types/netmask": "2.0.6", + "@types/node": "24.12.4", + "@types/sinon": "21.0.0", + "ip-address": "10.2.0", "ip6addr": "0.2.5", - "jest": "28.1.3", - "prettier": "2.7.1", - "semantic-release": "19.0.5", - "sinon": "14.0.0", - "ts-jest": "28.0.7", - "typescript": "4.7.4" - }, - "repository": { - "type": "git", - "url": "https://github.com/runk/node-maxmind.git" + "jest": "30.4.2", + "prettier": "3.8.3", + "semantic-release": "25.0.3", + "sinon": "21.0.3", + "typescript": "6.0.3" }, + "repository": "https://github.com/runk/node-maxmind", "bugs": { "mail": "deadrunk@gmail.com", "url": "http://github.com/runk/node-maxmind/issues" @@ -65,8 +64,10 @@ "scripts": { "build": "rm -rf lib/* && tsc", "format": "prettier --write .", - "prepublish": "npm run build", "semantic-release": "semantic-release", "test": "jest" + }, + "publishConfig": { + "provenance": true } } diff --git a/renovate.json b/renovate.json index b9c9af19..719a2cd9 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,6 @@ { - "extends": ["config:base"], + "extends": ["config:base", ":disableDependencyDashboard"], + "schedule": "on the first day of the month", "packageRules": [ { "updateTypes": ["minor", "patch", "pin", "digest"], diff --git a/src/__test__/integration.test.ts b/src/__test__/integration.test.ts index a05b138e..f737ac81 100755 --- a/src/__test__/integration.test.ts +++ b/src/__test__/integration.test.ts @@ -97,10 +97,10 @@ describe('maxmind', () => { float: 1.100000023841858, int32: -268435456, map: { mapX: { arrayX: [7, 8, 9], utf8_stringX: 'hello' } }, - uint128: '1329227995784915872903807060280344576', + uint128: 1329227995784915872903807060280344576n, uint16: 100, uint32: 268435456, - uint64: '1152921504606846976', + uint64: 1152921504606846976n, utf8_string: 'unicode! ☯ - ♫', }); }); @@ -117,10 +117,10 @@ describe('maxmind', () => { float: 0, int32: 0, map: {}, - uint128: 0, + uint128: 0n, uint16: 0, uint32: 0, - uint64: 0, + uint64: 0n, utf8_string: '', }); }); @@ -221,10 +221,10 @@ describe('maxmind', () => { utf8_stringX: 'hello', }, }, - uint128: '1329227995784915872903807060280344576', + uint128: 1329227995784915872903807060280344576n, uint16: 0x64, uint32: 268435456, - uint64: '1152921504606846976', + uint64: 1152921504606846976n, utf8_string: 'unicode! ☯ - ♫', }; const tests = [ diff --git a/src/fs.ts b/src/fs.ts index 9e648488..0d256a4f 100644 --- a/src/fs.ts +++ b/src/fs.ts @@ -5,4 +5,6 @@ export default { existsSync: fs.existsSync, readFile: util.promisify(fs.readFile), watchFile: fs.watchFile, + createReadStream: fs.createReadStream, + stat: util.promisify(fs.stat), }; diff --git a/src/index.test.ts b/src/index.test.ts index 001c1342..95069c95 100644 --- a/src/index.test.ts +++ b/src/index.test.ts @@ -18,6 +18,7 @@ describe('index', () => { sandbox.stub(fs, 'watchFile').callsFake((paramA, paramB, cb) => { watchHandler = cb; }); + sandbox.spy(fs, 'createReadStream'); sandbox.spy(fs, 'readFile'); }); afterEach(() => { @@ -113,7 +114,7 @@ describe('index', () => { // it('should check for an error when cannot read database on update', async () => { // var counter = 0; // var cb = function(err, reader) { - // // Indeed couter is kinda gross. + // // Indeed counter is kinda gross. // switch (counter++) { // case 0: // assert.strictEqual(err, null); diff --git a/src/index.ts b/src/index.ts index 10550912..7f3e185e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,11 +1,14 @@ import assert from 'assert'; -import lru from 'tiny-lru'; -import { Reader } from 'mmdb-lib'; +import { Reader, Response } from 'mmdb-lib'; +import { lru } from 'tiny-lru'; import fs from './fs'; import ip from './ip'; import isGzip from './is-gzip'; import utils from './utils'; +const LARGE_FILE_THRESHOLD = 512 * 1024 * 1024; +const STREAM_WATERMARK = 8 * 1024 * 1024; + type Callback = () => void; export interface OpenOpts { @@ -17,14 +20,63 @@ export interface OpenOpts { watchForUpdatesHook?: Callback; } -export const open = async ( +/** + * Read large file in chunks. + * + * Reason it's not used for all file sizes is that it's slower than fs.readFile and uses + * a bit more memory due to the buffer operations. + * + * Node seems to have a limit of 2GB for fs.readFileSync, so we need to use streams for + * larger files. + * + * @param filepath + * @param size + * @returns + */ +const readLargeFile = async (filepath: string, size: number): Promise => + new Promise((resolve, reject) => { + let buffer = Buffer.allocUnsafe(size); + let offset = 0; + const stream = fs.createReadStream(filepath, { + highWaterMark: STREAM_WATERMARK, + }); + + stream.on('data', (chunk: string | Buffer) => { + if (Buffer.isBuffer(chunk)) { + chunk.copy(buffer, offset); + offset += chunk.length; + } else { + const bufferChunk = Buffer.from(chunk); + bufferChunk.copy(buffer, offset); + offset += bufferChunk.length; + } + }); + + stream.on('end', () => { + stream.close(); + resolve(buffer); + }); + + stream.on('error', (err) => { + reject(err); + }); + }); + +const readFile = async (filepath: string): Promise => { + const fstat = await fs.stat(filepath); + return fstat.size < LARGE_FILE_THRESHOLD + ? fs.readFile(filepath) + : readLargeFile(filepath, fstat.size); +}; + +export const open = async ( filepath: string, opts?: OpenOpts, cb?: Callback ): Promise> => { assert(!cb, utils.legacyErrorMessage); - const database = await fs.readFile(filepath); + const database = await readFile(filepath); if (isGzip(database)) { throw new Error( @@ -32,7 +84,7 @@ export const open = async ( ); } - const cache = lru((opts && opts.cache && opts.cache.max) || 6000); + const cache = lru(opts?.cache?.max || 10_000); const reader = new Reader(database, { cache }); if (opts && !!opts.watchForUpdates) { @@ -63,7 +115,7 @@ export const open = async ( if (!(await waitExists())) { return; } - const updatedDatabase = await fs.readFile(filepath); + const updatedDatabase = await readFile(filepath); cache.clear(); reader.load(updatedDatabase); if (opts.watchForUpdatesHook) { diff --git a/tsconfig.json b/tsconfig.json index 43e91a75..200b63e5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,14 +8,15 @@ "declaration": true, "diagnostics": true, "esModuleInterop": true, - "extendedDiagnostics": true, + "extendedDiagnostics": false, "listEmittedFiles": true, "module": "commonjs", "removeComments": true, "sourceMap": true, "strict": true, - "lib": ["ES2019"], - "target": "ES2019" + "types": ["node"], + "lib": ["ES2020"], + "target": "ES2020" }, "exclude": [ "__mocks__",