From dd082660b02511c1068c494da96fa99f3a97d12e Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 17 Feb 2023 19:27:32 +0800 Subject: [PATCH 01/15] Support node 12 --- deno.json | 6 +++ .../node/pagination_js_commonjs/example.js | 40 +++++++++++-------- scripts/build_npm.ts | 8 +++- src/utils.ts | 17 ++++++-- tests/utils_test.ts | 1 + 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/deno.json b/deno.json index 0aadab2..e5c46d3 100644 --- a/deno.json +++ b/deno.json @@ -16,5 +16,11 @@ "files": { "exclude": ["npm/", "examples/node"] } + }, + "compilerOptions": { + "lib": [ + "dom", + "deno.ns" + ] } } diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_commonjs/example.js index c0b5065..5b6d02a 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_commonjs/example.js @@ -18,11 +18,13 @@ const run = async () => { "First page links", extractLinks(page1.organic_results), ); - let page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); + if (page1.next) { + let page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + } // Pagination (callback) getJson("google", params, (page1) => { @@ -30,22 +32,26 @@ const run = async () => { "First page links", extractLinks(page1.organic_results), ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); + if (page1.next) { + page1.next((page2) => { + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + }); + } }); // Use global config config.api_key = apiKey; page1 = await getJson("google", { q: "Coffee" }); - page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); + if (page1.next) { + page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + } // Pagination loop (async/await) let links = []; @@ -53,7 +59,7 @@ const run = async () => { while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; - page = await page.next?.(); + page = page.next ? await page.next(): undefined; } console.log(links); diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 3bfb545..b145b90 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -4,6 +4,7 @@ import { version } from "../version.ts"; await emptyDir("./npm"); await build({ + test: false, // Turned off to avoid publishing tests entryPoints: ["./mod.ts"], rootTestDir: "./tests", outDir: "./npm", @@ -16,8 +17,11 @@ await build({ // https://deno.land/std/async/delay.ts relies on DOMException. // This is only used in tests. domException: "dev", // Only used in tests. - - undici: true, // Required for `fetch` + }, + compilerOptions: { + // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping + lib: ["es2019"], + target: "ES2019", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 1d9bd02..8868a53 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ import type { EngineMap } from "./engines/engine_map.ts"; import { version } from "../version.ts"; +import fetch from "npm:cross-fetch@3.1.4"; type UrlParameters = Record< string, @@ -111,7 +112,7 @@ export function buildUrl

( return `${_internals.getBaseUrl()}${path}?${searchParams}`; } -export async function execute

( +export function execute

( path: string, parameters: P, timeout: number, @@ -120,7 +121,17 @@ export async function execute

( ...parameters, source: getSource(), }); - return await _internals.fetch(url, { - signal: AbortSignal.timeout(timeout), + // https://github.com/github/fetch/issues/175#issuecomment-216791333 + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Timeout")), timeout); + _internals.fetch(url) + .then((res) => { + clearTimeout(timer); + resolve(res); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }); }); } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 8e27b11..f12c2b3 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -23,6 +23,7 @@ import { extractNextParameters, haveParametersChanged, } from "../src/utils.ts"; +import { Response } from "npm:cross-fetch@3.1.4"; loadSync({ export: true }); const BASE_URL = Deno.env.get("ENV_TYPE") === "local" From f2df3db4ddfa04a88ca2352f78667eb870705f4c Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 16 Feb 2023 16:58:02 +0800 Subject: [PATCH 02/15] Support node 10 --- scripts/build_npm.ts | 4 ++-- src/utils.ts | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index b145b90..7820f8f 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -20,8 +20,8 @@ await build({ }, compilerOptions: { // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping - lib: ["es2019"], - target: "ES2019", + lib: ["es2018"], + target: "ES2018", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 8868a53..0e528b8 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -54,8 +54,11 @@ export function extractNextParameters< if (nextUrlString) { const nextUrl = new URL(nextUrlString); - const nextParameters = Object.fromEntries(nextUrl.searchParams.entries()); - delete nextParameters["engine"]; + const nextParameters: Record = {}; + for (const [k, v] of nextUrl.searchParams.entries()) { + if (k === "engine") continue; + nextParameters[k] = v; + } return nextParameters as NextParameters; } } From 2ca76af96e3f1095c27fd6c471ed570c6eb94720 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 16 Feb 2023 20:02:29 +0800 Subject: [PATCH 03/15] Support node 7 --- examples/node/pagination_js_commonjs/example.js | 10 +++++----- scripts/build_npm.ts | 5 +++-- src/utils.ts | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_commonjs/example.js index 5b6d02a..f3b048f 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_commonjs/example.js @@ -16,13 +16,13 @@ const run = async () => { let page1 = await getJson("google", params); console.log( "First page links", - extractLinks(page1.organic_results), + extractLinks(page1.organic_results) ); if (page1.next) { let page2 = await page1.next(); console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); } @@ -30,13 +30,13 @@ const run = async () => { getJson("google", params, (page1) => { console.log( "First page links", - extractLinks(page1.organic_results), + extractLinks(page1.organic_results) ); if (page1.next) { page1.next((page2) => { console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); }); } @@ -49,7 +49,7 @@ const run = async () => { page2 = await page1.next(); console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); } diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 7820f8f..2a268da 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -5,6 +5,7 @@ await emptyDir("./npm"); await build({ test: false, // Turned off to avoid publishing tests + typeCheck: false, entryPoints: ["./mod.ts"], rootTestDir: "./tests", outDir: "./npm", @@ -20,8 +21,8 @@ await build({ }, compilerOptions: { // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping - lib: ["es2018"], - target: "ES2018", + lib: ["es2017"], + target: "ES2017", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 0e528b8..16c8b82 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,8 @@ import type { EngineMap } from "./engines/engine_map.ts"; import { version } from "../version.ts"; import fetch from "npm:cross-fetch@3.1.4"; +import core from "npm:core-js-pure@3.28.0"; +const { globalThis, URL, URLSearchParams } = core; type UrlParameters = Record< string, From 5d1a4f9e55cb7b2f6fd6e036f14514ecda69dc9d Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 17 Feb 2023 18:35:04 +0800 Subject: [PATCH 04/15] Update examples --- .../.env.example | 0 .../{basic_js_esm => basic_js_node_14_up}/README.md | 4 ++-- .../{basic_js_esm => basic_js_node_14_up}/example.js | 8 ++++++++ .../{basic_js_esm => basic_js_node_14_up}/package.json | 0 .../{basic_js_esm => basic_js_node_7_up}/.env.example | 0 .../README.md | 4 ++-- .../example.js | 4 ++++ .../package.json | 0 .../{basic_ts_esm => basic_ts_node_14_up}/.env.example | 0 .../README.md | 4 ++-- .../{basic_ts_esm => basic_ts_node_14_up}/example.ts | 8 ++++++++ .../{basic_ts_esm => basic_ts_node_14_up}/package.json | 0 .../tsconfig.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.js | 10 ++++++++++ .../package.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.js | 4 ++++ .../package.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.ts | 10 ++++++++++ .../package.json | 0 .../tsconfig.json | 0 26 files changed, 56 insertions(+), 12 deletions(-) rename examples/node/{basic_js_commonjs => basic_js_node_14_up}/.env.example (100%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/README.md (88%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/example.js (59%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/package.json (100%) rename examples/node/{basic_js_esm => basic_js_node_7_up}/.env.example (100%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/README.md (88%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/example.js (92%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/package.json (100%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/.env.example (100%) rename examples/node/{pagination_ts_esm => basic_ts_node_14_up}/README.md (89%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/example.ts (60%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/package.json (100%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/tsconfig.json (100%) rename examples/node/{pagination_js_commonjs => pagination_js_node_14_up}/.env.example (100%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/README.md (87%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/example.js (71%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/package.json (100%) rename examples/node/{pagination_js_esm => pagination_js_node_7_up}/.env.example (100%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/README.md (86%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/example.js (97%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/package.json (100%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/.env.example (100%) rename examples/node/{basic_ts_esm => pagination_ts_node_14_up}/README.md (88%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/example.ts (72%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/package.json (100%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/tsconfig.json (100%) diff --git a/examples/node/basic_js_commonjs/.env.example b/examples/node/basic_js_node_14_up/.env.example similarity index 100% rename from examples/node/basic_js_commonjs/.env.example rename to examples/node/basic_js_node_14_up/.env.example diff --git a/examples/node/basic_js_esm/README.md b/examples/node/basic_js_node_14_up/README.md similarity index 88% rename from examples/node/basic_js_esm/README.md rename to examples/node/basic_js_node_14_up/README.md index aed73cb..2818395 100644 --- a/examples/node/basic_js_esm/README.md +++ b/examples/node/basic_js_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic JavaScript (ESM) example +# Basic JavaScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_js_esm +cd examples/node/basic_js_node_14_up npm i npm start ``` diff --git a/examples/node/basic_js_esm/example.js b/examples/node/basic_js_node_14_up/example.js similarity index 59% rename from examples/node/basic_js_esm/example.js rename to examples/node/basic_js_node_14_up/example.js index caab132..bb171f7 100644 --- a/examples/node/basic_js_esm/example.js +++ b/examples/node/basic_js_node_14_up/example.js @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/basic_js_esm/package.json b/examples/node/basic_js_node_14_up/package.json similarity index 100% rename from examples/node/basic_js_esm/package.json rename to examples/node/basic_js_node_14_up/package.json diff --git a/examples/node/basic_js_esm/.env.example b/examples/node/basic_js_node_7_up/.env.example similarity index 100% rename from examples/node/basic_js_esm/.env.example rename to examples/node/basic_js_node_7_up/.env.example diff --git a/examples/node/basic_js_commonjs/README.md b/examples/node/basic_js_node_7_up/README.md similarity index 88% rename from examples/node/basic_js_commonjs/README.md rename to examples/node/basic_js_node_7_up/README.md index 2ded14b..3b73b0c 100644 --- a/examples/node/basic_js_commonjs/README.md +++ b/examples/node/basic_js_node_7_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic JavaScript (CommonJS) example +# Basic JavaScript example for Node.js 7 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_js_commonjs +cd examples/node/basic_js_node_7_up npm i npm start ``` diff --git a/examples/node/basic_js_commonjs/example.js b/examples/node/basic_js_node_7_up/example.js similarity index 92% rename from examples/node/basic_js_commonjs/example.js rename to examples/node/basic_js_node_7_up/example.js index 43136a8..301baf4 100644 --- a/examples/node/basic_js_commonjs/example.js +++ b/examples/node/basic_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); diff --git a/examples/node/basic_js_commonjs/package.json b/examples/node/basic_js_node_7_up/package.json similarity index 100% rename from examples/node/basic_js_commonjs/package.json rename to examples/node/basic_js_node_7_up/package.json diff --git a/examples/node/basic_ts_esm/.env.example b/examples/node/basic_ts_node_14_up/.env.example similarity index 100% rename from examples/node/basic_ts_esm/.env.example rename to examples/node/basic_ts_node_14_up/.env.example diff --git a/examples/node/pagination_ts_esm/README.md b/examples/node/basic_ts_node_14_up/README.md similarity index 89% rename from examples/node/pagination_ts_esm/README.md rename to examples/node/basic_ts_node_14_up/README.md index 96e2230..44fd48a 100644 --- a/examples/node/pagination_ts_esm/README.md +++ b/examples/node/basic_ts_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination TypeScript (ESM) example +# Basic TypeScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_ts_esm +cd examples/node/basic_ts_node_14_up npm i npm start ``` diff --git a/examples/node/basic_ts_esm/example.ts b/examples/node/basic_ts_node_14_up/example.ts similarity index 60% rename from examples/node/basic_ts_esm/example.ts rename to examples/node/basic_ts_node_14_up/example.ts index 7faa58b..09b35b6 100644 --- a/examples/node/basic_ts_esm/example.ts +++ b/examples/node/basic_ts_node_14_up/example.ts @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/basic_ts_esm/package.json b/examples/node/basic_ts_node_14_up/package.json similarity index 100% rename from examples/node/basic_ts_esm/package.json rename to examples/node/basic_ts_node_14_up/package.json diff --git a/examples/node/basic_ts_esm/tsconfig.json b/examples/node/basic_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/basic_ts_esm/tsconfig.json rename to examples/node/basic_ts_node_14_up/tsconfig.json diff --git a/examples/node/pagination_js_commonjs/.env.example b/examples/node/pagination_js_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_js_commonjs/.env.example rename to examples/node/pagination_js_node_14_up/.env.example diff --git a/examples/node/pagination_js_esm/README.md b/examples/node/pagination_js_node_14_up/README.md similarity index 87% rename from examples/node/pagination_js_esm/README.md rename to examples/node/pagination_js_node_14_up/README.md index 4e4ebe2..81a9eb1 100644 --- a/examples/node/pagination_js_esm/README.md +++ b/examples/node/pagination_js_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination JavaScript (ESM) example +# Pagination JavaScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_js_esm +cd examples/node/pagination_js_node_14_up npm i npm start ``` diff --git a/examples/node/pagination_js_esm/example.js b/examples/node/pagination_js_node_14_up/example.js similarity index 71% rename from examples/node/pagination_js_esm/example.js rename to examples/node/pagination_js_node_14_up/example.js index d5bf869..650a3e5 100644 --- a/examples/node/pagination_js_esm/example.js +++ b/examples/node/pagination_js_node_14_up/example.js @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/pagination_js_esm/package.json b/examples/node/pagination_js_node_14_up/package.json similarity index 100% rename from examples/node/pagination_js_esm/package.json rename to examples/node/pagination_js_node_14_up/package.json diff --git a/examples/node/pagination_js_esm/.env.example b/examples/node/pagination_js_node_7_up/.env.example similarity index 100% rename from examples/node/pagination_js_esm/.env.example rename to examples/node/pagination_js_node_7_up/.env.example diff --git a/examples/node/pagination_js_commonjs/README.md b/examples/node/pagination_js_node_7_up/README.md similarity index 86% rename from examples/node/pagination_js_commonjs/README.md rename to examples/node/pagination_js_node_7_up/README.md index 0afbe7a..c5e9810 100644 --- a/examples/node/pagination_js_commonjs/README.md +++ b/examples/node/pagination_js_node_7_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination JavaScript (CommonJS) example +# Pagination JavaScript example for Node.js 7 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_js_commonjs +cd examples/node/pagination_js_node_7_up npm i npm start ``` diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_node_7_up/example.js similarity index 97% rename from examples/node/pagination_js_commonjs/example.js rename to examples/node/pagination_js_node_7_up/example.js index f3b048f..f998b37 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); diff --git a/examples/node/pagination_js_commonjs/package.json b/examples/node/pagination_js_node_7_up/package.json similarity index 100% rename from examples/node/pagination_js_commonjs/package.json rename to examples/node/pagination_js_node_7_up/package.json diff --git a/examples/node/pagination_ts_esm/.env.example b/examples/node/pagination_ts_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_ts_esm/.env.example rename to examples/node/pagination_ts_node_14_up/.env.example diff --git a/examples/node/basic_ts_esm/README.md b/examples/node/pagination_ts_node_14_up/README.md similarity index 88% rename from examples/node/basic_ts_esm/README.md rename to examples/node/pagination_ts_node_14_up/README.md index 3c2ec7e..c95d849 100644 --- a/examples/node/basic_ts_esm/README.md +++ b/examples/node/pagination_ts_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic TypeScript (ESM) example +# Pagination TypeScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_ts_esm +cd examples/node/pagination_ts_node_14_up npm i npm start ``` diff --git a/examples/node/pagination_ts_esm/example.ts b/examples/node/pagination_ts_node_14_up/example.ts similarity index 72% rename from examples/node/pagination_ts_esm/example.ts rename to examples/node/pagination_ts_node_14_up/example.ts index cb5858d..7d3ec91 100644 --- a/examples/node/pagination_ts_esm/example.ts +++ b/examples/node/pagination_ts_node_14_up/example.ts @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/pagination_ts_esm/package.json b/examples/node/pagination_ts_node_14_up/package.json similarity index 100% rename from examples/node/pagination_ts_esm/package.json rename to examples/node/pagination_ts_node_14_up/package.json diff --git a/examples/node/pagination_ts_esm/tsconfig.json b/examples/node/pagination_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/pagination_ts_esm/tsconfig.json rename to examples/node/pagination_ts_node_14_up/tsconfig.json From 36f6ba6e46dad436073d843ba4f908ff87e6fa4d Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 24 Feb 2023 19:42:18 +0800 Subject: [PATCH 05/15] Update docs --- CHANGELOG.md | 2 ++ README.md | 33 +++++++++++++++++-- ...ating_from_google_search_results_nodejs.md | 2 -- 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 098518c..d4a2eca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to ### Added +- Add support for Node.js 7.10.1 and newer. + ### Changed - Remove `settings.json`, update CONTRIBUTING.md. diff --git a/README.md b/README.md index 469f571..9056db9 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,39 @@ more. ### Node.js -Ensure you're running at least Node.js v16.14. +- Supports Node.js 7.10.1 and newer. +- Refer to [this example](examples/node/basic_js_node_7_up) for help. ```bash npm install serpapi ``` +```js +const { getJson } = require("serpapi"); +getJson("google", { + api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key + q: "coffee", + location: "Austin, Texas", +}, (json) => { + console.log(json["organic_results"]); +}); +``` + +### Node.js with ES Modules (ESM) and top-level await + +- If you prefer using the `import` syntax and top-level `await`, you need to use + at least Node.js 14.8.0. +- Refer to [this example](examples/node/basic_js_node_14_up) for help. + +You will need to add `"type": "module"` to your `package.json`: + +```js +{ + "type": "module", + // rest of package.json +} +``` + ```js import { getJson } from "serpapi"; const response = await getJson("google", { @@ -35,7 +62,9 @@ console.log(response); ### Deno -Import directly from deno.land. Usage is otherwise the same as above. +- Import directly from deno.land. +- Usage is otherwise the same as above. +- Refer to [this example](examples/deno/basic_ts) for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index 2e869ab..6ef7f5d 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -51,8 +51,6 @@ migrate over to the `serpapi` npm package. - The `buildUrl`, `execute` and `search` methods are removed. Use `getJson` and `getHtml` functions instead. - The `SerpApiSearch` class is removed as a public class. -- Dropped support for Node.js 16.13 and below. This module supports Node.js - 16.14 and above. ## Fixed From 95be440ae0df55f2cc681e82b99db929e732bb82 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 27 Feb 2023 12:45:42 +0800 Subject: [PATCH 06/15] Update examples to depend on published npm module --- examples/node/basic_js_node_14_up/README.md | 29 ++-------------- .../node/basic_js_node_14_up/package.json | 2 +- examples/node/basic_js_node_7_up/README.md | 28 ++-------------- examples/node/basic_js_node_7_up/package.json | 2 +- examples/node/basic_ts_node_14_up/README.md | 33 ++----------------- .../node/basic_ts_node_14_up/package.json | 2 +- .../node/pagination_js_node_14_up/README.md | 29 ++-------------- .../pagination_js_node_14_up/package.json | 2 +- .../node/pagination_js_node_7_up/README.md | 28 ++-------------- .../node/pagination_js_node_7_up/package.json | 2 +- .../node/pagination_ts_node_14_up/README.md | 33 ++----------------- .../pagination_ts_node_14_up/package.json | 2 +- 12 files changed, 18 insertions(+), 174 deletions(-) diff --git a/examples/node/basic_js_node_14_up/README.md b/examples/node/basic_js_node_14_up/README.md index 2818395..3b36e40 100644 --- a/examples/node/basic_js_node_14_up/README.md +++ b/examples/node/basic_js_node_14_up/README.md @@ -2,39 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_js_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_node_14_up/package.json b/examples/node/basic_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/basic_js_node_14_up/package.json +++ b/examples/node/basic_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_js_node_7_up/README.md b/examples/node/basic_js_node_7_up/README.md index 3b73b0c..9085457 100644 --- a/examples/node/basic_js_node_7_up/README.md +++ b/examples/node/basic_js_node_7_up/README.md @@ -2,38 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_js_node_7_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_node_7_up/package.json b/examples/node/basic_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/basic_js_node_7_up/package.json +++ b/examples/node/basic_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_ts_node_14_up/README.md b/examples/node/basic_ts_node_14_up/README.md index 44fd48a..20e12ba 100644 --- a/examples/node/basic_ts_node_14_up/README.md +++ b/examples/node/basic_ts_node_14_up/README.md @@ -2,43 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_ts_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/basic_ts_node_14_up/package.json b/examples/node/basic_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/basic_ts_node_14_up/package.json +++ b/examples/node/basic_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", diff --git a/examples/node/pagination_js_node_14_up/README.md b/examples/node/pagination_js_node_14_up/README.md index 81a9eb1..d49b924 100644 --- a/examples/node/pagination_js_node_14_up/README.md +++ b/examples/node/pagination_js_node_14_up/README.md @@ -2,39 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_js_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_node_14_up/package.json b/examples/node/pagination_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/pagination_js_node_14_up/package.json +++ b/examples/node/pagination_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_js_node_7_up/README.md b/examples/node/pagination_js_node_7_up/README.md index c5e9810..4542a22 100644 --- a/examples/node/pagination_js_node_7_up/README.md +++ b/examples/node/pagination_js_node_7_up/README.md @@ -2,38 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_js_node_7_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_node_7_up/package.json b/examples/node/pagination_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/pagination_js_node_7_up/package.json +++ b/examples/node/pagination_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_ts_node_14_up/README.md b/examples/node/pagination_ts_node_14_up/README.md index c95d849..2f6cb33 100644 --- a/examples/node/pagination_ts_node_14_up/README.md +++ b/examples/node/pagination_ts_node_14_up/README.md @@ -2,43 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_ts_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/pagination_ts_node_14_up/package.json b/examples/node/pagination_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/pagination_ts_node_14_up/package.json +++ b/examples/node/pagination_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", From 4b94bb75ac125e991be52cf0e79a14bcaf6bef76 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 27 Feb 2023 12:46:58 +0800 Subject: [PATCH 07/15] Add docs on running examples on local files --- CONTRIBUTING.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29bbfa4..5c54dee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,25 @@ deno task test:cov # Get test coverage by running tests that hit "localhost" deno task test:ci # Run tests that hit "https://serpapi.com" ``` +## Run examples on local source files + +To run [examples](./examples/) on you local source files, follow these steps. + +1. Run `deno task npm` to build the files. +2. Update the respective example's `package.json` to depend on the local + `serpapi` module instead, + ```json + { + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node example.js" + } + } + ``` + ## Update documentation - Every exposed function must have associated JSDoc comments. @@ -112,7 +131,8 @@ deno task docs:gen TypeScript types are generated from the backend code. Follow these steps to update the types. -1. Run `bundle exec rails sdk:generate_ts_types` in the backend repository. +1. Run `bundle exec rails libraries:generate_ts_types` in the backend + repository. 2. Replace everything in `src/engines` with the generated files from `tmp/ts/engines`. 3. Update `mod.ts` with the new engine exports from `tmp/ts/mod.ts`. From 6ad7d5ef39cb6d31e4c41cab2621a5a3e94ed32f Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 11:23:13 +0800 Subject: [PATCH 08/15] Smoke tests --- .github/workflows/build.yml | 38 +++++++++- CONTRIBUTING.md | 3 +- deno.json | 10 ++- tests/smoke/.env.example | 1 + tests/smoke/.gitignore | 3 + tests/smoke/cjs.js | 136 ++++++++++++++++++++++++++++++++++++ tests/smoke/esm.js | 126 +++++++++++++++++++++++++++++++++ tests/smoke/package.json | 10 +++ 8 files changed, 323 insertions(+), 4 deletions(-) create mode 100644 tests/smoke/.env.example create mode 100644 tests/smoke/.gitignore create mode 100644 tests/smoke/cjs.js create mode 100644 tests/smoke/esm.js create mode 100644 tests/smoke/package.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 880d6f8..d888f50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ on: push jobs: build: + name: "Deno tests and npm build" runs-on: ubuntu-22.04 steps: - name: Checkout repo @@ -28,8 +29,43 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: '18.x' + node-version: '18.x' # Build files using a fixed node version registry-url: 'https://registry.npmjs.org' - name: Test building of npm files run: deno task npm + + - name: Upload build files for smoke tests + uses: actions/upload-artifact@v3 + with: + name: npm + path: npm/ + + smoke-tests: + name: "Smoke tests for different Node.js versions" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [18.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Run smoke tests + run: | + echo ${{ matrix.node-version }} + cd smoke + npm i + npm run esm \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c54dee..f87cc62 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,8 @@ If you use VSCode, use the following settings (`.vscode/settings.json`): "mod.ts", "version.ts", "src", - "tests", + "tests/*.ts", + "tests/engines/", "scripts", "examples/deno" ], diff --git a/deno.json b/deno.json index e5c46d3..6e8fb32 100644 --- a/deno.json +++ b/deno.json @@ -9,12 +9,18 @@ }, "fmt": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] } }, "lint": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] + } + }, + "test": { + "files": { + "include": ["tests/"], + "exclude": ["tests/smoke/"] } }, "compilerOptions": { diff --git a/tests/smoke/.env.example b/tests/smoke/.env.example new file mode 100644 index 0000000..fefbd78 --- /dev/null +++ b/tests/smoke/.env.example @@ -0,0 +1 @@ +API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/.gitignore b/tests/smoke/.gitignore new file mode 100644 index 0000000..217965e --- /dev/null +++ b/tests/smoke/.gitignore @@ -0,0 +1,3 @@ +# Smoke tests run on different node versions which generate different +# package-lock.json files. +package-lock.json \ No newline at end of file diff --git a/tests/smoke/cjs.js b/tests/smoke/cjs.js new file mode 100644 index 0000000..0fa2331 --- /dev/null +++ b/tests/smoke/cjs.js @@ -0,0 +1,136 @@ +/** + * Smoke test works for Node.js 7 and newer. + */ + +const Dotenv = require("dotenv"); +const { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } = require("serpapi"); + +Dotenv.config(); +const apiKey = process.env.API_KEY; + +const run = async () => { + console.log("running", process.versions.node); + + const params = { + q: "Coffee", + api_key: apiKey, + }; + + let searchId; + + // getJson async await + { + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + // getJson callback + { + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + page1.next((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + } + }); + } + + // getJson using global config + { + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + // getJson pagination loop (async/await) + { + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = page.next ? await page.next(): undefined; + } + if (links.length < 30) throw new Error("Incorrect number of links"); + } + + // getJson pagination loop (callback) + { + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); + } + + // getHtml + { + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + // getJsonBySearchId + { + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); + } + + // getHtmlBySearchId + { + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + // getAccount + { + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); + } + + // getLocations + { + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); + } + + console.log("success", process.versions.node); +}; + +run(); \ No newline at end of file diff --git a/tests/smoke/esm.js b/tests/smoke/esm.js new file mode 100644 index 0000000..405f2b0 --- /dev/null +++ b/tests/smoke/esm.js @@ -0,0 +1,126 @@ +/** + * Smoke test works for Node.js 14 and newer. + */ + +import Dotenv from "dotenv"; +import { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } from "serpapi"; + +Dotenv.config(); +const apiKey = process.env.API_KEY; + +console.log("running", process.versions.node); + +const params = { + q: "Coffee", + api_key: apiKey, +}; + +let searchId; + +// getJson async await +{ + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +// getJson callback +{ + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + page1.next?.((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + }); +} + +// getJson using global config +{ + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +// getJson pagination loop (async/await) +{ + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = await page.next?.(); + } + if (links.length < 30) throw new Error("Incorrect number of links"); +} + +// getJson pagination loop (callback) +{ + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); +} + +// getHtml +{ + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +// getJsonBySearchId +{ + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); +} + +// getHtmlBySearchId +{ + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +// getAccount +{ + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); +} + +// getLocations +{ + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); +} + +console.log("success", process.versions.node); diff --git a/tests/smoke/package.json b/tests/smoke/package.json new file mode 100644 index 0000000..d372141 --- /dev/null +++ b/tests/smoke/package.json @@ -0,0 +1,10 @@ +{ + "dependencies": { + "dotenv": "*", + "serpapi": "../../npm" + }, + "scripts": { + "cjs": "node cjs.js", + "esm": "node esm.js" + } +} From 6baab9ca930a10df1fefea513cd8f98e82698cbd Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 11:36:38 +0800 Subject: [PATCH 09/15] Refine build workflow --- .github/workflows/build.yml | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d888f50..ba11802 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,16 +15,16 @@ jobs: with: deno-version: v1.x - - name: Check formatting - run: deno fmt --check + # - name: Check formatting + # run: deno fmt --check - - name: Run linter - run: deno lint + # - name: Run linter + # run: deno lint - - name: Run tests - env: - SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} - run: deno task test:ci + # - name: Run tests + # env: + # SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + # run: deno task test:ci - name: Setup Node uses: actions/setup-node@v3 @@ -35,11 +35,15 @@ jobs: - name: Test building of npm files run: deno task npm + - name: Zip build files + run: zip npm.zip ./npm -r + - name: Upload build files for smoke tests uses: actions/upload-artifact@v3 with: name: npm - path: npm/ + path: npm.zip + retention-days: 3 smoke-tests: name: "Smoke tests for different Node.js versions" @@ -62,10 +66,13 @@ jobs: uses: actions/download-artifact@v3 with: name: npm - + + - name: Unzip build files + run: unzip npm.zip + - name: Run smoke tests run: | echo ${{ matrix.node-version }} - cd smoke + cd tests/smoke npm i - npm run esm \ No newline at end of file + npm run esm From 227e2d04b7ea2193384c3db7b6ad71ed93530c63 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 11:45:20 +0800 Subject: [PATCH 10/15] Refine build workflow --- .github/workflows/build.yml | 8 ++++---- tests/smoke/{ => commonjs}/.env.example | 0 tests/smoke/{cjs.js => commonjs/commonjs.js} | 0 tests/smoke/commonjs/package.json | 9 +++++++++ tests/smoke/esm/.env.example | 1 + tests/smoke/{ => esm}/esm.js | 0 tests/smoke/esm/package.json | 10 ++++++++++ tests/smoke/package.json | 10 ---------- 8 files changed, 24 insertions(+), 14 deletions(-) rename tests/smoke/{ => commonjs}/.env.example (100%) rename tests/smoke/{cjs.js => commonjs/commonjs.js} (100%) create mode 100644 tests/smoke/commonjs/package.json create mode 100644 tests/smoke/esm/.env.example rename tests/smoke/{ => esm}/esm.js (100%) create mode 100644 tests/smoke/esm/package.json delete mode 100644 tests/smoke/package.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba11802..53055a1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,7 +43,7 @@ jobs: with: name: npm path: npm.zip - retention-days: 3 + retention-days: 1 smoke-tests: name: "Smoke tests for different Node.js versions" @@ -70,9 +70,9 @@ jobs: - name: Unzip build files run: unzip npm.zip - - name: Run smoke tests + - name: Run smoke tests (esm) run: | - echo ${{ matrix.node-version }} - cd tests/smoke + echo ${{ matrix.node-version >= 14 }} + cd tests/smoke/esm npm i npm run esm diff --git a/tests/smoke/.env.example b/tests/smoke/commonjs/.env.example similarity index 100% rename from tests/smoke/.env.example rename to tests/smoke/commonjs/.env.example diff --git a/tests/smoke/cjs.js b/tests/smoke/commonjs/commonjs.js similarity index 100% rename from tests/smoke/cjs.js rename to tests/smoke/commonjs/commonjs.js diff --git a/tests/smoke/commonjs/package.json b/tests/smoke/commonjs/package.json new file mode 100644 index 0000000..cc428ad --- /dev/null +++ b/tests/smoke/commonjs/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node commonjs.js" + } +} diff --git a/tests/smoke/esm/.env.example b/tests/smoke/esm/.env.example new file mode 100644 index 0000000..fefbd78 --- /dev/null +++ b/tests/smoke/esm/.env.example @@ -0,0 +1 @@ +API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/esm.js b/tests/smoke/esm/esm.js similarity index 100% rename from tests/smoke/esm.js rename to tests/smoke/esm/esm.js diff --git a/tests/smoke/esm/package.json b/tests/smoke/esm/package.json new file mode 100644 index 0000000..2efdf1f --- /dev/null +++ b/tests/smoke/esm/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node esm.js" + } +} diff --git a/tests/smoke/package.json b/tests/smoke/package.json deleted file mode 100644 index d372141..0000000 --- a/tests/smoke/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "dependencies": { - "dotenv": "*", - "serpapi": "../../npm" - }, - "scripts": { - "cjs": "node cjs.js", - "esm": "node esm.js" - } -} From 30d11b7edfef494e6ba0a8cb1c61e0a590e0b952 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 11:52:15 +0800 Subject: [PATCH 11/15] Refine build workflow --- .github/workflows/build.yml | 40 ++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53055a1..e7958c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,8 +45,8 @@ jobs: path: npm.zip retention-days: 1 - smoke-tests: - name: "Smoke tests for different Node.js versions" + smoke-tests-esm: + name: "Smoke tests (ESM)" needs: build runs-on: ubuntu-22.04 strategy: @@ -70,9 +70,39 @@ jobs: - name: Unzip build files run: unzip npm.zip - - name: Run smoke tests (esm) + - name: Run smoke tests run: | - echo ${{ matrix.node-version >= 14 }} cd tests/smoke/esm npm i - npm run esm + npm start + + smoke-tests-commonjs: + name: "Smoke tests (commonjs)" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [7.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Unzip build files + run: unzip npm.zip + + - name: Run smoke tests + run: | + cd tests/smoke/commonjs + npm i + npm start From cfd736f9fdc1f9ce0ce9dfa223f02f32f78a65cb Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 11:58:10 +0800 Subject: [PATCH 12/15] Refine build workflow --- .github/workflows/build.yml | 16 ++++++++-------- tests/smoke/commonjs/commonjs.js | 5 ++++- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7958c8..07614fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,13 +45,13 @@ jobs: path: npm.zip retention-days: 1 - smoke-tests-esm: - name: "Smoke tests (ESM)" + smoke-tests-commonjs: + name: "Smoke tests (CommonJS)" needs: build runs-on: ubuntu-22.04 strategy: matrix: - node-version: [18.x] + node-version: [7.x] steps: - name: Checkout repo uses: actions/checkout@v3 @@ -72,17 +72,17 @@ jobs: - name: Run smoke tests run: | - cd tests/smoke/esm + cd tests/smoke/commonjs npm i npm start - smoke-tests-commonjs: - name: "Smoke tests (commonjs)" + smoke-tests-esm: + name: "Smoke tests (ESM)" needs: build runs-on: ubuntu-22.04 strategy: matrix: - node-version: [7.x] + node-version: [18.x] steps: - name: Checkout repo uses: actions/checkout@v3 @@ -103,6 +103,6 @@ jobs: - name: Run smoke tests run: | - cd tests/smoke/commonjs + cd tests/smoke/esm npm i npm start diff --git a/tests/smoke/commonjs/commonjs.js b/tests/smoke/commonjs/commonjs.js index 0fa2331..81accd4 100644 --- a/tests/smoke/commonjs/commonjs.js +++ b/tests/smoke/commonjs/commonjs.js @@ -133,4 +133,7 @@ const run = async () => { console.log("success", process.versions.node); }; -run(); \ No newline at end of file +run().catch((e) => { + console.error(e); + process.exitCode = 1; +}); \ No newline at end of file From 744a86c88cb00ef45e2f3a74f428225c1f37fed6 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 12:03:09 +0800 Subject: [PATCH 13/15] Refine build workflow --- .github/workflows/build.yml | 4 ++++ tests/smoke/commonjs/.env.example | 2 +- tests/smoke/commonjs/commonjs.js | 2 +- tests/smoke/esm/.env.example | 2 +- tests/smoke/esm/esm.js | 2 +- 5 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 07614fd..255c51a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,6 +71,8 @@ jobs: run: unzip npm.zip - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} run: | cd tests/smoke/commonjs npm i @@ -102,6 +104,8 @@ jobs: run: unzip npm.zip - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} run: | cd tests/smoke/esm npm i diff --git a/tests/smoke/commonjs/.env.example b/tests/smoke/commonjs/.env.example index fefbd78..6debcc7 100644 --- a/tests/smoke/commonjs/.env.example +++ b/tests/smoke/commonjs/.env.example @@ -1 +1 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/commonjs/commonjs.js b/tests/smoke/commonjs/commonjs.js index 81accd4..d925c7c 100644 --- a/tests/smoke/commonjs/commonjs.js +++ b/tests/smoke/commonjs/commonjs.js @@ -6,7 +6,7 @@ const Dotenv = require("dotenv"); const { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } = require("serpapi"); Dotenv.config(); -const apiKey = process.env.API_KEY; +const apiKey = process.env.SERPAPI_TEST_KEY; const run = async () => { console.log("running", process.versions.node); diff --git a/tests/smoke/esm/.env.example b/tests/smoke/esm/.env.example index fefbd78..6debcc7 100644 --- a/tests/smoke/esm/.env.example +++ b/tests/smoke/esm/.env.example @@ -1 +1 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/esm/esm.js b/tests/smoke/esm/esm.js index 405f2b0..c076469 100644 --- a/tests/smoke/esm/esm.js +++ b/tests/smoke/esm/esm.js @@ -6,7 +6,7 @@ import Dotenv from "dotenv"; import { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } from "serpapi"; Dotenv.config(); -const apiKey = process.env.API_KEY; +const apiKey = process.env.SERPAPI_TEST_KEY; console.log("running", process.versions.node); From 3a25d30eeef3e30dbe2fce2a9b11fdda49162967 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 12:08:34 +0800 Subject: [PATCH 14/15] Full run --- .github/workflows/build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 255c51a..513ed2f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,16 +15,16 @@ jobs: with: deno-version: v1.x - # - name: Check formatting - # run: deno fmt --check + - name: Check formatting + run: deno fmt --check - # - name: Run linter - # run: deno lint + - name: Run linter + run: deno lint - # - name: Run tests - # env: - # SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} - # run: deno task test:ci + - name: Run tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + run: deno task test:ci - name: Setup Node uses: actions/setup-node@v3 @@ -51,7 +51,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - node-version: [7.x] + node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] steps: - name: Checkout repo uses: actions/checkout@v3 @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - node-version: [18.x] + node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x] steps: - name: Checkout repo uses: actions/checkout@v3 From 4ab4937fb56df5118d2a32ce37d99d4168577878 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 12:14:55 +0800 Subject: [PATCH 15/15] Full run with more logging --- tests/smoke/commonjs/commonjs.js | 20 ++++++++++---------- tests/smoke/esm/esm.js | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/smoke/commonjs/commonjs.js b/tests/smoke/commonjs/commonjs.js index d925c7c..9cf60b1 100644 --- a/tests/smoke/commonjs/commonjs.js +++ b/tests/smoke/commonjs/commonjs.js @@ -18,8 +18,8 @@ const run = async () => { let searchId; - // getJson async await { + console.log("getJson async await"); const page1 = await getJson("google", params); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); @@ -29,8 +29,8 @@ const run = async () => { } } - // getJson callback { + console.log("getJson callback"); getJson("google", params, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { @@ -41,8 +41,8 @@ const run = async () => { }); } - // getJson using global config { + console.log("getJson using global config"); config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); @@ -52,8 +52,8 @@ const run = async () => { } } - // getJson pagination loop (async/await) { + console.log("getJson pagination loop (async/await)"); const links = []; let page = await getJson("google", params); while (page) { @@ -64,8 +64,8 @@ const run = async () => { if (links.length < 30) throw new Error("Incorrect number of links"); } - // getJson pagination loop (callback) { + console.log("getJson pagination loop (callback)"); const links = []; getJson("google", params, (page) => { links.push(...page.organic_results.map(r => r.link)); @@ -77,8 +77,8 @@ const run = async () => { }); } - // getHtml { + console.log("getHtml"); const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); @@ -87,8 +87,8 @@ const run = async () => { }); } - // getJsonBySearchId { + console.log("getJsonBySearchId"); config.api_key = apiKey; const json = await getJsonBySearchId(searchId); if (!json["organic_results"]) throw new Error("No organic results"); @@ -98,8 +98,8 @@ const run = async () => { }); } - // getHtmlBySearchId { + console.log("getHtmlBySearchId"); config.api_key = apiKey; const html = await getHtmlBySearchId(searchId); if (html.length < 1000) throw new Error("Incorrect HTML"); @@ -109,8 +109,8 @@ const run = async () => { }); } - // getAccount { + console.log("getAccount"); config.api_key = apiKey; const info = await getAccount(); if (!info["account_email"]) throw new Error("Incorrect account info"); @@ -120,8 +120,8 @@ const run = async () => { }); } - // getLocations { + console.log("getLocations"); const locations = await getLocations({ limit: 3 }); if (locations.length !== 3) throw new Error("Incorrect locations length"); diff --git a/tests/smoke/esm/esm.js b/tests/smoke/esm/esm.js index c076469..d6d494c 100644 --- a/tests/smoke/esm/esm.js +++ b/tests/smoke/esm/esm.js @@ -17,8 +17,8 @@ const params = { let searchId; -// getJson async await { + console.log("getJson async await") const page1 = await getJson("google", params); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); @@ -26,8 +26,8 @@ let searchId; if (!page2["organic_results"]) throw new Error("No organic results"); } -// getJson callback { + console.log("getJson callback") getJson("google", params, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); page1.next?.((page2) => { @@ -36,8 +36,8 @@ let searchId; }); } -// getJson using global config { + console.log("getJson using global config") config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); @@ -45,8 +45,8 @@ let searchId; if (!page2["organic_results"]) throw new Error("No organic results"); } -// getJson pagination loop (async/await) { + console.log("getJson pagination loop (async/await)") const links = []; let page = await getJson("google", params); while (page) { @@ -57,8 +57,8 @@ let searchId; if (links.length < 30) throw new Error("Incorrect number of links"); } -// getJson pagination loop (callback) { + console.log("getJson pagination loop (callback)") const links = []; getJson("google", params, (page) => { links.push(...page.organic_results.map(r => r.link)); @@ -70,8 +70,8 @@ let searchId; }); } -// getHtml { + console.log("getHtml") const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); @@ -80,8 +80,8 @@ let searchId; }); } -// getJsonBySearchId { + console.log("getJsonBySearchId") config.api_key = apiKey; const json = await getJsonBySearchId(searchId); if (!json["organic_results"]) throw new Error("No organic results"); @@ -91,8 +91,8 @@ let searchId; }); } -// getHtmlBySearchId { + console.log("getHtmlBySearchId") config.api_key = apiKey; const html = await getHtmlBySearchId(searchId); if (html.length < 1000) throw new Error("Incorrect HTML"); @@ -102,8 +102,8 @@ let searchId; }); } -// getAccount { + console.log("getAccount") config.api_key = apiKey; const info = await getAccount(); if (!info["account_email"]) throw new Error("Incorrect account info"); @@ -113,8 +113,8 @@ let searchId; }); } -// getLocations { + console.log("getLocations") const locations = await getLocations({ limit: 3 }); if (locations.length !== 3) throw new Error("Incorrect locations length");