diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 880d6f8..513ed2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ on: push jobs: build: + name: "Deno tests and npm build" runs-on: ubuntu-22.04 steps: - name: Checkout repo @@ -28,8 +29,84 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: '18.x' + node-version: '18.x' # Build files using a fixed node version registry-url: 'https://registry.npmjs.org' - name: Test building of npm files run: deno task npm + + - name: Zip build files + run: zip npm.zip ./npm -r + + - name: Upload build files for smoke tests + uses: actions/upload-artifact@v3 + with: + name: npm + path: npm.zip + retention-days: 1 + + smoke-tests-commonjs: + name: "Smoke tests (CommonJS)" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Unzip build files + run: unzip npm.zip + + - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + run: | + cd tests/smoke/commonjs + npm i + npm start + + smoke-tests-esm: + name: "Smoke tests (ESM)" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Unzip build files + run: unzip npm.zip + + - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + run: | + cd tests/smoke/esm + npm i + npm start diff --git a/CHANGELOG.md b/CHANGELOG.md index 098518c..d4a2eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to ### Added +- Add support for Node.js 7.10.1 and newer. + ### Changed - Remove `settings.json`, update CONTRIBUTING.md. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29bbfa4..f87cc62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,8 @@ If you use VSCode, use the following settings (`.vscode/settings.json`): "mod.ts", "version.ts", "src", - "tests", + "tests/*.ts", + "tests/engines/", "scripts", "examples/deno" ], @@ -96,6 +97,25 @@ deno task test:cov # Get test coverage by running tests that hit "localhost" deno task test:ci # Run tests that hit "https://serpapi.com" ``` +## Run examples on local source files + +To run [examples](./examples/) on you local source files, follow these steps. + +1. Run `deno task npm` to build the files. +2. Update the respective example's `package.json` to depend on the local + `serpapi` module instead, + ```json + { + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node example.js" + } + } + ``` + ## Update documentation - Every exposed function must have associated JSDoc comments. @@ -112,7 +132,8 @@ deno task docs:gen TypeScript types are generated from the backend code. Follow these steps to update the types. -1. Run `bundle exec rails sdk:generate_ts_types` in the backend repository. +1. Run `bundle exec rails libraries:generate_ts_types` in the backend + repository. 2. Replace everything in `src/engines` with the generated files from `tmp/ts/engines`. 3. Update `mod.ts` with the new engine exports from `tmp/ts/mod.ts`. diff --git a/README.md b/README.md index 469f571..9056db9 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,39 @@ more. ### Node.js -Ensure you're running at least Node.js v16.14. +- Supports Node.js 7.10.1 and newer. +- Refer to [this example](examples/node/basic_js_node_7_up) for help. ```bash npm install serpapi ``` +```js +const { getJson } = require("serpapi"); +getJson("google", { + api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key + q: "coffee", + location: "Austin, Texas", +}, (json) => { + console.log(json["organic_results"]); +}); +``` + +### Node.js with ES Modules (ESM) and top-level await + +- If you prefer using the `import` syntax and top-level `await`, you need to use + at least Node.js 14.8.0. +- Refer to [this example](examples/node/basic_js_node_14_up) for help. + +You will need to add `"type": "module"` to your `package.json`: + +```js +{ + "type": "module", + // rest of package.json +} +``` + ```js import { getJson } from "serpapi"; const response = await getJson("google", { @@ -35,7 +62,9 @@ console.log(response); ### Deno -Import directly from deno.land. Usage is otherwise the same as above. +- Import directly from deno.land. +- Usage is otherwise the same as above. +- Refer to [this example](examples/deno/basic_ts) for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; diff --git a/deno.json b/deno.json index 0aadab2..6e8fb32 100644 --- a/deno.json +++ b/deno.json @@ -9,12 +9,24 @@ }, "fmt": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] } }, "lint": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] } + }, + "test": { + "files": { + "include": ["tests/"], + "exclude": ["tests/smoke/"] + } + }, + "compilerOptions": { + "lib": [ + "dom", + "deno.ns" + ] } } diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index 2e869ab..6ef7f5d 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -51,8 +51,6 @@ migrate over to the `serpapi` npm package. - The `buildUrl`, `execute` and `search` methods are removed. Use `getJson` and `getHtml` functions instead. - The `SerpApiSearch` class is removed as a public class. -- Dropped support for Node.js 16.13 and below. This module supports Node.js - 16.14 and above. ## Fixed diff --git a/examples/node/basic_js_commonjs/README.md b/examples/node/basic_js_commonjs/README.md deleted file mode 100644 index 2ded14b..0000000 --- a/examples/node/basic_js_commonjs/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Node.js basic JavaScript (CommonJS) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/basic_js_commonjs -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_esm/README.md b/examples/node/basic_js_esm/README.md deleted file mode 100644 index aed73cb..0000000 --- a/examples/node/basic_js_esm/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Node.js basic JavaScript (ESM) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/basic_js_esm -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_commonjs/.env.example b/examples/node/basic_js_node_14_up/.env.example similarity index 100% rename from examples/node/basic_js_commonjs/.env.example rename to examples/node/basic_js_node_14_up/.env.example diff --git a/examples/node/basic_js_node_14_up/README.md b/examples/node/basic_js_node_14_up/README.md new file mode 100644 index 0000000..3b36e40 --- /dev/null +++ b/examples/node/basic_js_node_14_up/README.md @@ -0,0 +1,15 @@ +# Basic JavaScript example for Node.js 14 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/basic_js_esm/example.js b/examples/node/basic_js_node_14_up/example.js similarity index 59% rename from examples/node/basic_js_esm/example.js rename to examples/node/basic_js_node_14_up/example.js index caab132..bb171f7 100644 --- a/examples/node/basic_js_esm/example.js +++ b/examples/node/basic_js_node_14_up/example.js @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/basic_js_esm/package.json b/examples/node/basic_js_node_14_up/package.json similarity index 79% rename from examples/node/basic_js_esm/package.json rename to examples/node/basic_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/basic_js_esm/package.json +++ b/examples/node/basic_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_js_esm/.env.example b/examples/node/basic_js_node_7_up/.env.example similarity index 100% rename from examples/node/basic_js_esm/.env.example rename to examples/node/basic_js_node_7_up/.env.example diff --git a/examples/node/basic_js_node_7_up/README.md b/examples/node/basic_js_node_7_up/README.md new file mode 100644 index 0000000..9085457 --- /dev/null +++ b/examples/node/basic_js_node_7_up/README.md @@ -0,0 +1,15 @@ +# Basic JavaScript example for Node.js 7 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/basic_js_commonjs/example.js b/examples/node/basic_js_node_7_up/example.js similarity index 92% rename from examples/node/basic_js_commonjs/example.js rename to examples/node/basic_js_node_7_up/example.js index 43136a8..301baf4 100644 --- a/examples/node/basic_js_commonjs/example.js +++ b/examples/node/basic_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); diff --git a/examples/node/basic_js_commonjs/package.json b/examples/node/basic_js_node_7_up/package.json similarity index 76% rename from examples/node/basic_js_commonjs/package.json rename to examples/node/basic_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/basic_js_commonjs/package.json +++ b/examples/node/basic_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_ts_esm/README.md b/examples/node/basic_ts_esm/README.md deleted file mode 100644 index 3c2ec7e..0000000 --- a/examples/node/basic_ts_esm/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Node.js basic TypeScript (ESM) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/basic_ts_esm -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/basic_ts_esm/.env.example b/examples/node/basic_ts_node_14_up/.env.example similarity index 100% rename from examples/node/basic_ts_esm/.env.example rename to examples/node/basic_ts_node_14_up/.env.example diff --git a/examples/node/basic_ts_node_14_up/README.md b/examples/node/basic_ts_node_14_up/README.md new file mode 100644 index 0000000..20e12ba --- /dev/null +++ b/examples/node/basic_ts_node_14_up/README.md @@ -0,0 +1,15 @@ +# Basic TypeScript example for Node.js 14 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/basic_ts_esm/example.ts b/examples/node/basic_ts_node_14_up/example.ts similarity index 60% rename from examples/node/basic_ts_esm/example.ts rename to examples/node/basic_ts_node_14_up/example.ts index 7faa58b..09b35b6 100644 --- a/examples/node/basic_ts_esm/example.ts +++ b/examples/node/basic_ts_node_14_up/example.ts @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/basic_ts_esm/package.json b/examples/node/basic_ts_node_14_up/package.json similarity index 86% rename from examples/node/basic_ts_esm/package.json rename to examples/node/basic_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/basic_ts_esm/package.json +++ b/examples/node/basic_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", diff --git a/examples/node/basic_ts_esm/tsconfig.json b/examples/node/basic_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/basic_ts_esm/tsconfig.json rename to examples/node/basic_ts_node_14_up/tsconfig.json diff --git a/examples/node/pagination_js_commonjs/README.md b/examples/node/pagination_js_commonjs/README.md deleted file mode 100644 index 0afbe7a..0000000 --- a/examples/node/pagination_js_commonjs/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Node.js pagination JavaScript (CommonJS) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/pagination_js_commonjs -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_esm/README.md b/examples/node/pagination_js_esm/README.md deleted file mode 100644 index 4e4ebe2..0000000 --- a/examples/node/pagination_js_esm/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Node.js pagination JavaScript (ESM) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/pagination_js_esm -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_commonjs/.env.example b/examples/node/pagination_js_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_js_commonjs/.env.example rename to examples/node/pagination_js_node_14_up/.env.example diff --git a/examples/node/pagination_js_node_14_up/README.md b/examples/node/pagination_js_node_14_up/README.md new file mode 100644 index 0000000..d49b924 --- /dev/null +++ b/examples/node/pagination_js_node_14_up/README.md @@ -0,0 +1,15 @@ +# Pagination JavaScript example for Node.js 14 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/pagination_js_esm/example.js b/examples/node/pagination_js_node_14_up/example.js similarity index 71% rename from examples/node/pagination_js_esm/example.js rename to examples/node/pagination_js_node_14_up/example.js index d5bf869..650a3e5 100644 --- a/examples/node/pagination_js_esm/example.js +++ b/examples/node/pagination_js_node_14_up/example.js @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/pagination_js_esm/package.json b/examples/node/pagination_js_node_14_up/package.json similarity index 79% rename from examples/node/pagination_js_esm/package.json rename to examples/node/pagination_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/pagination_js_esm/package.json +++ b/examples/node/pagination_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_js_esm/.env.example b/examples/node/pagination_js_node_7_up/.env.example similarity index 100% rename from examples/node/pagination_js_esm/.env.example rename to examples/node/pagination_js_node_7_up/.env.example diff --git a/examples/node/pagination_js_node_7_up/README.md b/examples/node/pagination_js_node_7_up/README.md new file mode 100644 index 0000000..4542a22 --- /dev/null +++ b/examples/node/pagination_js_node_7_up/README.md @@ -0,0 +1,15 @@ +# Pagination JavaScript example for Node.js 7 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_node_7_up/example.js similarity index 63% rename from examples/node/pagination_js_commonjs/example.js rename to examples/node/pagination_js_node_7_up/example.js index c0b5065..f998b37 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); @@ -16,36 +20,42 @@ const run = async () => { let page1 = await getJson("google", params); console.log( "First page links", - extractLinks(page1.organic_results), - ); - let page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), + extractLinks(page1.organic_results) ); + if (page1.next) { + let page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results) + ); + } // Pagination (callback) getJson("google", params, (page1) => { console.log( "First page links", - extractLinks(page1.organic_results), + extractLinks(page1.organic_results) ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); + if (page1.next) { + page1.next((page2) => { + console.log( + "Second page links", + extractLinks(page2.organic_results) + ); + }); + } }); // Use global config config.api_key = apiKey; page1 = await getJson("google", { q: "Coffee" }); - page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); + if (page1.next) { + page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results) + ); + } // Pagination loop (async/await) let links = []; @@ -53,7 +63,7 @@ const run = async () => { while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; - page = await page.next?.(); + page = page.next ? await page.next(): undefined; } console.log(links); diff --git a/examples/node/pagination_js_commonjs/package.json b/examples/node/pagination_js_node_7_up/package.json similarity index 76% rename from examples/node/pagination_js_commonjs/package.json rename to examples/node/pagination_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/pagination_js_commonjs/package.json +++ b/examples/node/pagination_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_ts_esm/README.md b/examples/node/pagination_ts_esm/README.md deleted file mode 100644 index 96e2230..0000000 --- a/examples/node/pagination_ts_esm/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Node.js pagination TypeScript (ESM) example - -## Usage - -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -3. Install dependencies and run example - -```bash -cd examples/node/pagination_ts_esm -npm i -npm start -``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/pagination_ts_esm/.env.example b/examples/node/pagination_ts_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_ts_esm/.env.example rename to examples/node/pagination_ts_node_14_up/.env.example diff --git a/examples/node/pagination_ts_node_14_up/README.md b/examples/node/pagination_ts_node_14_up/README.md new file mode 100644 index 0000000..2f6cb33 --- /dev/null +++ b/examples/node/pagination_ts_node_14_up/README.md @@ -0,0 +1,15 @@ +# Pagination TypeScript example for Node.js 14 and newer + +## Usage + +1. Setup environment variables + +- Duplicate `.env.example` and name it `.env`. +- Replace `YOUR_API_KEY` with your SerpApi API key. + +2. Install dependencies and run example + +```bash +npm i +npm start +``` diff --git a/examples/node/pagination_ts_esm/example.ts b/examples/node/pagination_ts_node_14_up/example.ts similarity index 72% rename from examples/node/pagination_ts_esm/example.ts rename to examples/node/pagination_ts_node_14_up/example.ts index cb5858d..7d3ec91 100644 --- a/examples/node/pagination_ts_esm/example.ts +++ b/examples/node/pagination_ts_node_14_up/example.ts @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/pagination_ts_esm/package.json b/examples/node/pagination_ts_node_14_up/package.json similarity index 86% rename from examples/node/pagination_ts_esm/package.json rename to examples/node/pagination_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/pagination_ts_esm/package.json +++ b/examples/node/pagination_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", diff --git a/examples/node/pagination_ts_esm/tsconfig.json b/examples/node/pagination_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/pagination_ts_esm/tsconfig.json rename to examples/node/pagination_ts_node_14_up/tsconfig.json diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 3bfb545..2a268da 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -4,6 +4,8 @@ import { version } from "../version.ts"; await emptyDir("./npm"); await build({ + test: false, // Turned off to avoid publishing tests + typeCheck: false, entryPoints: ["./mod.ts"], rootTestDir: "./tests", outDir: "./npm", @@ -16,8 +18,11 @@ await build({ // https://deno.land/std/async/delay.ts relies on DOMException. // This is only used in tests. domException: "dev", // Only used in tests. - - undici: true, // Required for `fetch` + }, + compilerOptions: { + // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping + lib: ["es2017"], + target: "ES2017", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 1d9bd02..16c8b82 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,8 @@ import type { EngineMap } from "./engines/engine_map.ts"; import { version } from "../version.ts"; +import fetch from "npm:cross-fetch@3.1.4"; +import core from "npm:core-js-pure@3.28.0"; +const { globalThis, URL, URLSearchParams } = core; type UrlParameters = Record< string, @@ -53,8 +56,11 @@ export function extractNextParameters< if (nextUrlString) { const nextUrl = new URL(nextUrlString); - const nextParameters = Object.fromEntries(nextUrl.searchParams.entries()); - delete nextParameters["engine"]; + const nextParameters: Record = {}; + for (const [k, v] of nextUrl.searchParams.entries()) { + if (k === "engine") continue; + nextParameters[k] = v; + } return nextParameters as NextParameters; } } @@ -111,7 +117,7 @@ export function buildUrl

( return `${_internals.getBaseUrl()}${path}?${searchParams}`; } -export async function execute

( +export function execute

( path: string, parameters: P, timeout: number, @@ -120,7 +126,17 @@ export async function execute

( ...parameters, source: getSource(), }); - return await _internals.fetch(url, { - signal: AbortSignal.timeout(timeout), + // https://github.com/github/fetch/issues/175#issuecomment-216791333 + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Timeout")), timeout); + _internals.fetch(url) + .then((res) => { + clearTimeout(timer); + resolve(res); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }); }); } diff --git a/tests/smoke/.gitignore b/tests/smoke/.gitignore new file mode 100644 index 0000000..217965e --- /dev/null +++ b/tests/smoke/.gitignore @@ -0,0 +1,3 @@ +# Smoke tests run on different node versions which generate different +# package-lock.json files. +package-lock.json \ No newline at end of file diff --git a/tests/smoke/commonjs/.env.example b/tests/smoke/commonjs/.env.example new file mode 100644 index 0000000..6debcc7 --- /dev/null +++ b/tests/smoke/commonjs/.env.example @@ -0,0 +1 @@ +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/commonjs/commonjs.js b/tests/smoke/commonjs/commonjs.js new file mode 100644 index 0000000..9cf60b1 --- /dev/null +++ b/tests/smoke/commonjs/commonjs.js @@ -0,0 +1,139 @@ +/** + * Smoke test works for Node.js 7 and newer. + */ + +const Dotenv = require("dotenv"); +const { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } = require("serpapi"); + +Dotenv.config(); +const apiKey = process.env.SERPAPI_TEST_KEY; + +const run = async () => { + console.log("running", process.versions.node); + + const params = { + q: "Coffee", + api_key: apiKey, + }; + + let searchId; + + { + console.log("getJson async await"); + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson callback"); + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + page1.next((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + } + }); + } + + { + console.log("getJson using global config"); + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson pagination loop (async/await)"); + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = page.next ? await page.next(): undefined; + } + if (links.length < 30) throw new Error("Incorrect number of links"); + } + + { + console.log("getJson pagination loop (callback)"); + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); + } + + { + console.log("getHtml"); + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + { + console.log("getJsonBySearchId"); + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); + } + + { + console.log("getHtmlBySearchId"); + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + { + console.log("getAccount"); + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); + } + + { + console.log("getLocations"); + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); + } + + console.log("success", process.versions.node); +}; + +run().catch((e) => { + console.error(e); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/tests/smoke/commonjs/package.json b/tests/smoke/commonjs/package.json new file mode 100644 index 0000000..cc428ad --- /dev/null +++ b/tests/smoke/commonjs/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node commonjs.js" + } +} diff --git a/tests/smoke/esm/.env.example b/tests/smoke/esm/.env.example new file mode 100644 index 0000000..6debcc7 --- /dev/null +++ b/tests/smoke/esm/.env.example @@ -0,0 +1 @@ +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/esm/esm.js b/tests/smoke/esm/esm.js new file mode 100644 index 0000000..d6d494c --- /dev/null +++ b/tests/smoke/esm/esm.js @@ -0,0 +1,126 @@ +/** + * Smoke test works for Node.js 14 and newer. + */ + +import Dotenv from "dotenv"; +import { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } from "serpapi"; + +Dotenv.config(); +const apiKey = process.env.SERPAPI_TEST_KEY; + +console.log("running", process.versions.node); + +const params = { + q: "Coffee", + api_key: apiKey, +}; + +let searchId; + +{ + console.log("getJson async await") + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson callback") + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + page1.next?.((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + }); +} + +{ + console.log("getJson using global config") + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson pagination loop (async/await)") + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = await page.next?.(); + } + if (links.length < 30) throw new Error("Incorrect number of links"); +} + +{ + console.log("getJson pagination loop (callback)") + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); +} + +{ + console.log("getHtml") + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +{ + console.log("getJsonBySearchId") + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); +} + +{ + console.log("getHtmlBySearchId") + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +{ + console.log("getAccount") + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); +} + +{ + console.log("getLocations") + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); +} + +console.log("success", process.versions.node); diff --git a/tests/smoke/esm/package.json b/tests/smoke/esm/package.json new file mode 100644 index 0000000..2efdf1f --- /dev/null +++ b/tests/smoke/esm/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node esm.js" + } +} diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 8e27b11..f12c2b3 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -23,6 +23,7 @@ import { extractNextParameters, haveParametersChanged, } from "../src/utils.ts"; +import { Response } from "npm:cross-fetch@3.1.4"; loadSync({ export: true }); const BASE_URL = Deno.env.get("ENV_TYPE") === "local"