From bf650e13863e92bf053a8449adf7498b4c0dcae4 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Tue, 9 Jan 2024 11:40:06 +0000 Subject: [PATCH 1/5] Add multi-query support --- dist/query.js | 165 +++++++----- dist/update-repo-task-status.js | 2 - dist/update-repo-task-statuses.js | 2 - src/codeql.test.ts | 39 ++- src/codeql.ts | 252 ++++++++++++------ src/json-schemas/ResolvedQueries.json | 2 - src/query.ts | 10 +- .../test_pack_multiple_queries/.gitignore | 1 + .../test_pack_multiple_queries/qlpack.yml | 10 + .../test_pack_multiple_queries/x/query.ql | 5 + .../test_pack_multiple_queries/x/y/MyLib.qll | 1 + .../test_pack_multiple_queries/z/query.ql | 5 + .../test_pack_multiple_queries/z/y/MyLib.qll | 1 + 13 files changed, 341 insertions(+), 154 deletions(-) create mode 100644 testdata/test_pack_multiple_queries/.gitignore create mode 100644 testdata/test_pack_multiple_queries/qlpack.yml create mode 100644 testdata/test_pack_multiple_queries/x/query.ql create mode 100644 testdata/test_pack_multiple_queries/x/y/MyLib.qll create mode 100644 testdata/test_pack_multiple_queries/z/query.ql create mode 100644 testdata/test_pack_multiple_queries/z/y/MyLib.qll diff --git a/dist/query.js b/dist/query.js index b530bc589..cf8d2d190 100644 --- a/dist/query.js +++ b/dist/query.js @@ -74350,8 +74350,6 @@ var ResolvedQueries_default = { items: { type: "string" }, - maxItems: 1, - minItems: 1, type: "array" } } @@ -74803,47 +74801,74 @@ async function runQuery(codeql, database, nwo, queryPack) { databaseName, queryPackName ]); - const bqrsFilePath = import_path.default.join("results", "results.bqrs"); - const tempBqrsFilePath = getBqrsFile(databaseName); - import_fs2.default.renameSync(tempBqrsFilePath, bqrsFilePath); - const bqrsInfo = await getBqrsInfo(codeql, bqrsFilePath); - const compatibleQueryKinds = bqrsInfo.compatibleQueryKinds; - const queryMetadata = await getQueryMetadata( + const queryPaths = await getRemoteQueryPackQueries(codeql, queryPack); + const queryPackRunResults = await getQueryPackRunResults( codeql, - await getRemoteQueryPackDefaultQuery(codeql, queryPack) + databaseName, + queryPaths, + queryPack, + queryPackName ); const sourceLocationPrefix = await getSourceLocationPrefix(codeql); - const sarifOutputType = getSarifOutputType( - queryMetadata, - compatibleQueryKinds + const shouldGenerateSarif = await queryPackSupportsSarif( + codeql, + queryPackRunResults ); let resultCount; let sarifFilePath; - if (sarifOutputType !== void 0) { + if (shouldGenerateSarif) { const sarif = await generateSarif( codeql, - bqrsFilePath, nwo, - queryMetadata, - sarifOutputType, databaseName, - sourceLocationPrefix, + queryPackName, databaseSHA ); resultCount = getSarifResultCount(sarif); sarifFilePath = import_path.default.join("results", "results.sarif"); import_fs2.default.writeFileSync(sarifFilePath, JSON.stringify(sarif)); } else { - resultCount = getBqrsResultCount(bqrsInfo); + resultCount = queryPackRunResults.totalResultsCount; } + const bqrsFilePaths = await adjustBqrsFiles(queryPackRunResults); return { resultCount, databaseSHA, + databaseName, sourceLocationPrefix, - bqrsFilePath, + bqrsFilePaths, sarifFilePath }; } +async function adjustBqrsFiles(queryPackRunResults) { + if (queryPackRunResults.queries.length === 1) { + const currentBqrsFilePath = import_path.default.join( + queryPackRunResults.resultsBasePath, + queryPackRunResults.queries[0].relativeBqrsFilePath + ); + const newBqrsFilePath = import_path.default.join("results", "results.bqrs"); + await import_fs2.default.promises.rename(currentBqrsFilePath, newBqrsFilePath); + return [newBqrsFilePath]; + } + return await Promise.all( + queryPackRunResults.queries.map(async (query) => { + const newPath = await moveBqrsFileToResultsDir( + queryPackRunResults.resultsBasePath, + query.relativeBqrsFilePath + ); + return newPath; + }) + ); +} +async function moveBqrsFileToResultsDir(resultsBasePath, relativeBqrsFilePath) { + const oldPath = import_path.default.join(resultsBasePath, relativeBqrsFilePath); + const newPath = import_path.default.join("results", relativeBqrsFilePath); + if (!import_fs2.default.existsSync(import_path.default.dirname(newPath))) { + await import_fs2.default.promises.mkdir(import_path.default.dirname(newPath), { recursive: true }); + } + await import_fs2.default.promises.rename(oldPath, newPath); + return newPath; +} async function downloadDatabase(repoId, repoName, language, pat) { let authHeader = void 0; if (pat) { @@ -74913,6 +74938,54 @@ async function getSourceLocationPrefix(codeql) { ); return resolvedDatabase.sourceLocationPrefix; } +async function getQueryPackRunResults(codeql, databaseName, queryPaths, queryPackPath, queryPackName) { + const resultsBasePath = `${databaseName}/results`; + const queries = []; + let totalResultsCount = 0; + for (const queryPath of queryPaths) { + const queryPackRelativePath = import_path.default.relative(queryPackPath, queryPath); + const parsedQueryPath = import_path.default.parse(queryPackRelativePath); + const relativeBqrsFilePath = import_path.default.join( + queryPackName, + parsedQueryPath.dir, + `${parsedQueryPath.name}.bqrs` + ); + const bqrsFilePath = import_path.default.join(resultsBasePath, relativeBqrsFilePath); + if (!import_fs2.default.existsSync(bqrsFilePath)) { + throw new Error( + `Could not find BQRS file for query ${queryPath} at ${bqrsFilePath}` + ); + } + const bqrsInfo = await getBqrsInfo(codeql, bqrsFilePath); + queries.push({ + queryPath, + relativeBqrsFilePath, + bqrsInfo + }); + totalResultsCount += getBqrsResultCount(bqrsInfo); + } + return { + totalResultsCount, + resultsBasePath, + queries + }; +} +async function querySupportsSarif(codeql, queryPath, bqrsInfo) { + const compatibleQueryKinds = bqrsInfo.compatibleQueryKinds; + const queryMetadata = await getQueryMetadata(codeql, queryPath); + const sarifOutputType = getSarifOutputType( + queryMetadata, + compatibleQueryKinds + ); + return sarifOutputType !== void 0; +} +async function queryPackSupportsSarif(codeql, queriesResultInfo) { + return (await Promise.all( + queriesResultInfo.queries.map( + (q) => querySupportsSarif(codeql, q.queryPath, q.bqrsInfo) + ) + )).some((result) => result); +} function getSarifOutputType(queryMetadata, compatibleQueryKinds) { const queryKind = queryMetadata.kind; if (queryKind === "path-problem" && compatibleQueryKinds.includes("PathProblem")) { @@ -74923,32 +74996,17 @@ function getSarifOutputType(queryMetadata, compatibleQueryKinds) { return void 0; } } -async function generateSarif(codeql, bqrs, nwo, queryMetadata, sarifOutputType, databaseName, sourceLocationPrefix, databaseSHA) { - const { - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- we are explicitly excluding this since it's calculated separately - kind, - id: queryId, - ...passthroughQueryMetadata - } = queryMetadata; +async function generateSarif(codeql, nwo, databaseName, queryPackName, databaseSHA) { const sarifFile = import_path.default.join("results", "results.sarif"); await (0, import_exec.exec)(codeql, [ - "bqrs", - "interpret", + "database", + "interpret-results", "--format=sarif-latest", `--output=${sarifFile}`, - `-t=kind=${sarifOutputType}`, - `-t=id=${queryId || "remote-query"}`, - // Forward all of the query metadata. - ...Object.entries(passthroughQueryMetadata).map( - ([key, value]) => `-t=${key}=${value}` - ), "--sarif-add-snippets", "--no-group-results", - // Hard-coded the source archive as src.zip inside the database, since that's - // where the CLI puts it. If this changes, we need to update this path. - `--source-archive=${databaseName}/src.zip`, - `--source-location-prefix=${sourceLocationPrefix}`, - bqrs + databaseName, + queryPackName ]); const sarif = validateObject( JSON.parse(import_fs2.default.readFileSync(sarifFile, "utf8")), @@ -74998,7 +75056,7 @@ function getDatabaseMetadata(database) { return {}; } } -async function getRemoteQueryPackDefaultQuery(codeql, queryPack) { +async function getRemoteQueryPackQueries(codeql, queryPack) { const output = await (0, import_exec.getExecOutput)(codeql, [ "resolve", "queries", @@ -75007,25 +75065,7 @@ async function getRemoteQueryPackDefaultQuery(codeql, queryPack) { queryPack, getQueryPackName(queryPack) ]); - const queries = validateObject(JSON.parse(output.stdout), "resolvedQueries"); - return queries[0]; -} -function getBqrsFile(databaseName) { - let dbResultsFolder = `${databaseName}/results`; - let entries; - while ((entries = import_fs2.default.readdirSync(dbResultsFolder, { withFileTypes: true })) && entries.length === 1 && entries[0].isDirectory()) { - dbResultsFolder = import_path.default.join(dbResultsFolder, entries[0].name); - } - if (entries.length !== 1) { - throw new Error( - `Expected a single file in ${dbResultsFolder}, found: ${entries}` - ); - } - const entry = entries[0]; - if (!entry.isFile() || !entry.name.endsWith(".bqrs")) { - throw new Error(`Unexpected file in ${dbResultsFolder}: ${entry.name}`); - } - return import_path.default.join(dbResultsFolder, entry.name); + return validateObject(JSON.parse(output.stdout), "resolvedQueries"); } function getQueryPackName(queryPackPath) { const qlpackFile = import_path.default.join(queryPackPath, "qlpack.yml"); @@ -75153,8 +75193,11 @@ async function getArtifactContentsForUpload(runQueryResult) { const sarifFileContents = import_fs3.default.createReadStream(runQueryResult.sarifFilePath); zip.file("results.sarif", sarifFileContents); } - const bqrsFileContents = import_fs3.default.createReadStream(runQueryResult.bqrsFilePath); - zip.file("results.bqrs", bqrsFileContents); + for (const bqrsFilePath of runQueryResult.bqrsFilePaths) { + const bqrsFileContents = import_fs3.default.createReadStream(bqrsFilePath); + const relativePath = import_path2.default.relative("results", bqrsFilePath); + zip.file(relativePath, bqrsFileContents); + } return await zip.generateAsync({ compression: "DEFLATE", type: "nodebuffer" diff --git a/dist/update-repo-task-status.js b/dist/update-repo-task-status.js index 718d9e50c..b4001274a 100644 --- a/dist/update-repo-task-status.js +++ b/dist/update-repo-task-status.js @@ -44890,8 +44890,6 @@ var ResolvedQueries_default = { items: { type: "string" }, - maxItems: 1, - minItems: 1, type: "array" } } diff --git a/dist/update-repo-task-statuses.js b/dist/update-repo-task-statuses.js index 563e1dc44..bcbffda59 100644 --- a/dist/update-repo-task-statuses.js +++ b/dist/update-repo-task-statuses.js @@ -44891,8 +44891,6 @@ var ResolvedQueries_default = { items: { type: "string" }, - maxItems: 1, - minItems: 1, type: "array" } } diff --git a/src/codeql.test.ts b/src/codeql.test.ts index b5b08cdcf..105318026 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -10,7 +10,7 @@ import { getBqrsInfo, getDatabaseMetadata, BQRSInfo, - getRemoteQueryPackDefaultQuery, + getRemoteQueryPackQueries, injectVersionControlInfo, getSarifResultCount, Sarif, @@ -58,6 +58,7 @@ test("running a query in a pack", async (t) => { await runQuery("codeql", t.context.db, "a/b", queryPack); t.true(fs.existsSync(path.join("results", "results.bqrs"))); + t.false(fs.existsSync(path.join("results", "codeql/queries/x/query.bqrs"))); const bqrsInfo: BQRSInfo = await getBqrsInfo( "codeql", @@ -72,6 +73,37 @@ test("running a query in a pack", async (t) => { } }); +test("running multiple queries in a pack", async (t) => { + const queryPack = path.resolve("testdata/test_pack_multiple_queries"); + const tmpDir = fs.mkdtempSync("tmp"); + const cwd = process.cwd(); + process.chdir(tmpDir); + try { + await runQuery("codeql", t.context.db, "a/b", queryPack); + + const bqrsFilePath1 = "results/codeql/queries/x/query.bqrs"; + t.true(fs.existsSync(bqrsFilePath1)); + + const bqrsInfo1 = await getBqrsInfo("codeql", bqrsFilePath1); + t.is(1, bqrsInfo1.resultSets.length); + t.is("#select", bqrsInfo1.resultSets[0].name); + t.true(bqrsInfo1.compatibleQueryKinds.includes("Table")); + + const bqrsFilePath2 = "results/codeql/queries/z/query.bqrs"; + t.true(fs.existsSync(bqrsFilePath2)); + + const bqrsInfo2 = await getBqrsInfo("codeql", bqrsFilePath2); + t.is(1, bqrsInfo2.resultSets.length); + t.is("#select", bqrsInfo2.resultSets[0].name); + t.true(bqrsInfo2.compatibleQueryKinds.includes("Table")); + + t.false(fs.existsSync(path.join("results", "results.bqrs"))); + } finally { + process.chdir(cwd); + await rmRF(tmpDir); + } +}); + test("getting the commit SHA and CLI version from a database", async (t) => { const tmpDir = fs.mkdtempSync("tmp"); try { @@ -143,10 +175,9 @@ test("getting the commit SHA when the codeql-database.yml does not exist", async }); test("getting the default query from a pack", async (t) => { - t.is( - await getRemoteQueryPackDefaultQuery("codeql", "testdata/test_pack"), + t.deepEqual(await getRemoteQueryPackQueries("codeql", "testdata/test_pack"), [ path.resolve("testdata/test_pack/x/query.ql"), - ); + ]); }); test("populating the SARIF versionControlProvenance property", (t) => { diff --git a/src/codeql.ts b/src/codeql.ts index 7bfe6dd2a..932bb8a58 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -13,8 +13,9 @@ import { parseYamlFromFile } from "./yaml"; export interface RunQueryResult { resultCount: number; databaseSHA: string | undefined; + databaseName: string; sourceLocationPrefix: string; - bqrsFilePath: string; + bqrsFilePaths: string[]; sarifFilePath?: string; } @@ -76,51 +77,97 @@ export async function runQuery( queryPackName, ]); - const bqrsFilePath = path.join("results", "results.bqrs"); - const tempBqrsFilePath = getBqrsFile(databaseName); - fs.renameSync(tempBqrsFilePath, bqrsFilePath); + const queryPaths = await getRemoteQueryPackQueries(codeql, queryPack); - const bqrsInfo = await getBqrsInfo(codeql, bqrsFilePath); - const compatibleQueryKinds = bqrsInfo.compatibleQueryKinds; - const queryMetadata = await getQueryMetadata( + // Calculate query run information like BQRS file paths, etc. + const queryPackRunResults = await getQueryPackRunResults( codeql, - await getRemoteQueryPackDefaultQuery(codeql, queryPack), + databaseName, + queryPaths, + queryPack, + queryPackName, ); const sourceLocationPrefix = await getSourceLocationPrefix(codeql); - const sarifOutputType = getSarifOutputType( - queryMetadata, - compatibleQueryKinds, + + const shouldGenerateSarif = await queryPackSupportsSarif( + codeql, + queryPackRunResults, ); + let resultCount: number; let sarifFilePath: string | undefined; - if (sarifOutputType !== undefined) { + if (shouldGenerateSarif) { const sarif = await generateSarif( codeql, - bqrsFilePath, nwo, - queryMetadata, - sarifOutputType, databaseName, - sourceLocationPrefix, + queryPackName, databaseSHA, ); resultCount = getSarifResultCount(sarif); sarifFilePath = path.join("results", "results.sarif"); fs.writeFileSync(sarifFilePath, JSON.stringify(sarif)); } else { - resultCount = getBqrsResultCount(bqrsInfo); + resultCount = queryPackRunResults.totalResultsCount; } + const bqrsFilePaths = await adjustBqrsFiles(queryPackRunResults); + return { resultCount, databaseSHA, + databaseName, sourceLocationPrefix, - bqrsFilePath, + bqrsFilePaths, sarifFilePath, }; } +async function adjustBqrsFiles( + queryPackRunResults: QueryPackRunResults, +): Promise { + if (queryPackRunResults.queries.length === 1) { + // If we have a single query, move the BQRS file to "results.bqrs" in order to + // maintain backwards compatibility with the VS Code extension, since it expects + // the BQRS file to be at the top level and be called "results.bqrs". + const currentBqrsFilePath = path.join( + queryPackRunResults.resultsBasePath, + queryPackRunResults.queries[0].relativeBqrsFilePath, + ); + const newBqrsFilePath = path.join("results", "results.bqrs"); + await fs.promises.rename(currentBqrsFilePath, newBqrsFilePath); + return [newBqrsFilePath]; + } + + // If we have multiple queries, move the BQRS files to the correct location and + // return the new paths. + return await Promise.all( + queryPackRunResults.queries.map(async (query) => { + const newPath = await moveBqrsFileToResultsDir( + queryPackRunResults.resultsBasePath, + query.relativeBqrsFilePath, + ); + return newPath; + }), + ); +} + +async function moveBqrsFileToResultsDir( + resultsBasePath: string, + relativeBqrsFilePath: string, +): Promise { + const oldPath = path.join(resultsBasePath, relativeBqrsFilePath); + const newPath = path.join("results", relativeBqrsFilePath); + + if (!fs.existsSync(path.dirname(newPath))) { + await fs.promises.mkdir(path.dirname(newPath), { recursive: true }); + } + + await fs.promises.rename(oldPath, newPath); + return newPath; +} + export async function downloadDatabase( repoId: number, repoName: string, @@ -230,6 +277,102 @@ async function getSourceLocationPrefix(codeql: string) { return resolvedDatabase.sourceLocationPrefix; } +interface QueryPackRunResults { + queries: Array<{ + queryPath: string; + relativeBqrsFilePath: string; + bqrsInfo: BQRSInfo; + }>; + totalResultsCount: number; + resultsBasePath: string; +} + +async function getQueryPackRunResults( + codeql: string, + databaseName: string, + queryPaths: string[], + queryPackPath: string, + queryPackName: string, +): Promise { + // This is where results are saved, according to + // https://codeql.github.com/docs/codeql-cli/manual/database-run-queries/ + const resultsBasePath = `${databaseName}/results`; + + const queries: Array<{ + queryPath: string; + relativeBqrsFilePath: string; + bqrsInfo: BQRSInfo; + }> = []; + + let totalResultsCount = 0; + + for (const queryPath of queryPaths) { + // Calculate the BQRS file path + const queryPackRelativePath = path.relative(queryPackPath, queryPath); + const parsedQueryPath = path.parse(queryPackRelativePath); + const relativeBqrsFilePath = path.join( + queryPackName, + parsedQueryPath.dir, + `${parsedQueryPath.name}.bqrs`, + ); + const bqrsFilePath = path.join(resultsBasePath, relativeBqrsFilePath); + + if (!fs.existsSync(bqrsFilePath)) { + throw new Error( + `Could not find BQRS file for query ${queryPath} at ${bqrsFilePath}`, + ); + } + + const bqrsInfo = await getBqrsInfo(codeql, bqrsFilePath); + + queries.push({ + queryPath, + relativeBqrsFilePath, + bqrsInfo, + }); + + totalResultsCount += getBqrsResultCount(bqrsInfo); + } + + return { + totalResultsCount, + resultsBasePath, + queries, + }; +} + +async function querySupportsSarif( + codeql: string, + queryPath: string, + bqrsInfo: BQRSInfo, +): Promise { + const compatibleQueryKinds = bqrsInfo.compatibleQueryKinds; + + const queryMetadata = await getQueryMetadata(codeql, queryPath); + + const sarifOutputType = getSarifOutputType( + queryMetadata, + compatibleQueryKinds, + ); + + return sarifOutputType !== undefined; +} + +async function queryPackSupportsSarif( + codeql: string, + queriesResultInfo: QueryPackRunResults, +) { + // Some queries in the pack must support SARIF in order + // for the query pack to support SARIF. + return ( + await Promise.all( + queriesResultInfo.queries.map((q) => + querySupportsSarif(codeql, q.queryPath, q.bqrsInfo), + ), + ) + ).some((result) => result); +} + /** * Checks if the query kind is compatible with SARIF output. */ @@ -256,40 +399,21 @@ export function getSarifOutputType( // Generates sarif from the given bqrs file, if query kind supports it async function generateSarif( codeql: string, - bqrs: string, nwo: string, - queryMetadata: QueryMetadata, - sarifOutputType: SarifOutputType, databaseName: string, - sourceLocationPrefix: string, + queryPackName: string, databaseSHA?: string, ): Promise { - const { - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- we are explicitly excluding this since it's calculated separately - kind, - id: queryId, - ...passthroughQueryMetadata - } = queryMetadata; - const sarifFile = path.join("results", "results.sarif"); await exec(codeql, [ - "bqrs", - "interpret", + "database", + "interpret-results", "--format=sarif-latest", `--output=${sarifFile}`, - `-t=kind=${sarifOutputType}`, - `-t=id=${queryId || "remote-query"}`, - // Forward all of the query metadata. - ...Object.entries(passthroughQueryMetadata).map( - ([key, value]) => `-t=${key}=${value}`, - ), "--sarif-add-snippets", "--no-group-results", - // Hard-coded the source archive as src.zip inside the database, since that's - // where the CLI puts it. If this changes, we need to update this path. - `--source-archive=${databaseName}/src.zip`, - `--source-location-prefix=${sourceLocationPrefix}`, - bqrs, + databaseName, + queryPackName, ]); const sarif = validateObject( JSON.parse(fs.readFileSync(sarifFile, "utf8")), @@ -377,19 +501,19 @@ export function getDatabaseMetadata(database: string): DatabaseMetadata { } // The expected output from "codeql resolve queries" in getRemoteQueryPackDefaultQuery -export type ResolvedQueries = [string]; +export type ResolvedQueries = string[]; /** - * Gets the query for a pack, assuming there is a single query in that pack's default suite. + * Gets the queries for a pack. * * @param codeql The path to the codeql CLI * @param queryPack The path to the query pack on disk. * @returns The path to a query file. */ -export async function getRemoteQueryPackDefaultQuery( +export async function getRemoteQueryPackQueries( codeql: string, queryPack: string, -): Promise { +): Promise { const output = await getExecOutput(codeql, [ "resolve", "queries", @@ -399,41 +523,7 @@ export async function getRemoteQueryPackDefaultQuery( getQueryPackName(queryPack), ]); - const queries = validateObject(JSON.parse(output.stdout), "resolvedQueries"); - return queries[0]; -} - -/** - * Finds the BQRS result file for a database and ensures that exactly one is produced. - * Returns the path to that BQRS file. - * @param databaseName The name of the database that was analyzed. - * @returns string The path to the BQRS result file. - */ -function getBqrsFile(databaseName: string): string { - // This is where results are saved, according to - // https://codeql.github.com/docs/codeql-cli/manual/database-run-queries/ - let dbResultsFolder = `${databaseName}/results`; - let entries: fs.Dirent[]; - while ( - (entries = fs.readdirSync(dbResultsFolder, { withFileTypes: true })) && - entries.length === 1 && - entries[0].isDirectory() - ) { - dbResultsFolder = path.join(dbResultsFolder, entries[0].name); - } - - if (entries.length !== 1) { - throw new Error( - `Expected a single file in ${dbResultsFolder}, found: ${entries}`, - ); - } - - const entry = entries[0]; - if (!entry.isFile() || !entry.name.endsWith(".bqrs")) { - throw new Error(`Unexpected file in ${dbResultsFolder}: ${entry.name}`); - } - - return path.join(dbResultsFolder, entry.name); + return validateObject(JSON.parse(output.stdout), "resolvedQueries"); } function getQueryPackName(queryPackPath: string) { diff --git a/src/json-schemas/ResolvedQueries.json b/src/json-schemas/ResolvedQueries.json index c1562e161..d3b9b150d 100644 --- a/src/json-schemas/ResolvedQueries.json +++ b/src/json-schemas/ResolvedQueries.json @@ -6,8 +6,6 @@ "items": { "type": "string" }, - "maxItems": 1, - "minItems": 1, "type": "array" } } diff --git a/src/query.ts b/src/query.ts index b397e7e1a..3e03cca63 100644 --- a/src/query.ts +++ b/src/query.ts @@ -157,8 +157,14 @@ async function getArtifactContentsForUpload( const sarifFileContents = fs.createReadStream(runQueryResult.sarifFilePath); zip.file("results.sarif", sarifFileContents); } - const bqrsFileContents = fs.createReadStream(runQueryResult.bqrsFilePath); - zip.file("results.bqrs", bqrsFileContents); + + for (const bqrsFilePath of runQueryResult.bqrsFilePaths) { + // All the files are in the results dir, so we need to make sure + // that the zip does not contain a "results" directory. + const bqrsFileContents = fs.createReadStream(bqrsFilePath); + const relativePath = path.relative("results", bqrsFilePath); + zip.file(relativePath, bqrsFileContents); + } return await zip.generateAsync({ compression: "DEFLATE", diff --git a/testdata/test_pack_multiple_queries/.gitignore b/testdata/test_pack_multiple_queries/.gitignore new file mode 100644 index 000000000..988107fe1 --- /dev/null +++ b/testdata/test_pack_multiple_queries/.gitignore @@ -0,0 +1 @@ +.cache/ \ No newline at end of file diff --git a/testdata/test_pack_multiple_queries/qlpack.yml b/testdata/test_pack_multiple_queries/qlpack.yml new file mode 100644 index 000000000..88b9025f4 --- /dev/null +++ b/testdata/test_pack_multiple_queries/qlpack.yml @@ -0,0 +1,10 @@ +library: false +name: codeql/queries +version: 1.0.0 +buildMetadata: + creationTime: 2021-11-23T16:09:11.452312400Z + cliVersion: 2.7.2 +defaultSuite: + - description: Query suite for remote query + - query: x/query.ql + - query: z/query.ql diff --git a/testdata/test_pack_multiple_queries/x/query.ql b/testdata/test_pack_multiple_queries/x/query.ql new file mode 100644 index 000000000..870480618 --- /dev/null +++ b/testdata/test_pack_multiple_queries/x/query.ql @@ -0,0 +1,5 @@ +import y.MyLib + +from int i +where i in [0 .. 2] +select i, smallPlusOne(i) diff --git a/testdata/test_pack_multiple_queries/x/y/MyLib.qll b/testdata/test_pack_multiple_queries/x/y/MyLib.qll new file mode 100644 index 000000000..2a07f6237 --- /dev/null +++ b/testdata/test_pack_multiple_queries/x/y/MyLib.qll @@ -0,0 +1 @@ +int smallPlusOne(int i) { result = i + 1 and i in [0 .. 9] } diff --git a/testdata/test_pack_multiple_queries/z/query.ql b/testdata/test_pack_multiple_queries/z/query.ql new file mode 100644 index 000000000..870480618 --- /dev/null +++ b/testdata/test_pack_multiple_queries/z/query.ql @@ -0,0 +1,5 @@ +import y.MyLib + +from int i +where i in [0 .. 2] +select i, smallPlusOne(i) diff --git a/testdata/test_pack_multiple_queries/z/y/MyLib.qll b/testdata/test_pack_multiple_queries/z/y/MyLib.qll new file mode 100644 index 000000000..2a07f6237 --- /dev/null +++ b/testdata/test_pack_multiple_queries/z/y/MyLib.qll @@ -0,0 +1 @@ +int smallPlusOne(int i) { result = i + 1 and i in [0 .. 9] } From a92c28bbaa098302c6a13146122378379d207f56 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Wed, 10 Jan 2024 12:01:55 +0000 Subject: [PATCH 2/5] Rename getRemoteQueryPackQueries -> getQueryPackQueries --- dist/query.js | 4 ++-- src/codeql.test.ts | 4 ++-- src/codeql.ts | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dist/query.js b/dist/query.js index cf8d2d190..469587cb8 100644 --- a/dist/query.js +++ b/dist/query.js @@ -74801,7 +74801,7 @@ async function runQuery(codeql, database, nwo, queryPack) { databaseName, queryPackName ]); - const queryPaths = await getRemoteQueryPackQueries(codeql, queryPack); + const queryPaths = await getQueryPackQueries(codeql, queryPack); const queryPackRunResults = await getQueryPackRunResults( codeql, databaseName, @@ -75056,7 +75056,7 @@ function getDatabaseMetadata(database) { return {}; } } -async function getRemoteQueryPackQueries(codeql, queryPack) { +async function getQueryPackQueries(codeql, queryPack) { const output = await (0, import_exec.getExecOutput)(codeql, [ "resolve", "queries", diff --git a/src/codeql.test.ts b/src/codeql.test.ts index 105318026..5c21949c7 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -10,7 +10,7 @@ import { getBqrsInfo, getDatabaseMetadata, BQRSInfo, - getRemoteQueryPackQueries, + getQueryPackQueries, injectVersionControlInfo, getSarifResultCount, Sarif, @@ -175,7 +175,7 @@ test("getting the commit SHA when the codeql-database.yml does not exist", async }); test("getting the default query from a pack", async (t) => { - t.deepEqual(await getRemoteQueryPackQueries("codeql", "testdata/test_pack"), [ + t.deepEqual(await getQueryPackQueries("codeql", "testdata/test_pack"), [ path.resolve("testdata/test_pack/x/query.ql"), ]); }); diff --git a/src/codeql.ts b/src/codeql.ts index 932bb8a58..46ae510c1 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -77,7 +77,7 @@ export async function runQuery( queryPackName, ]); - const queryPaths = await getRemoteQueryPackQueries(codeql, queryPack); + const queryPaths = await getQueryPackQueries(codeql, queryPack); // Calculate query run information like BQRS file paths, etc. const queryPackRunResults = await getQueryPackRunResults( @@ -500,7 +500,7 @@ export function getDatabaseMetadata(database: string): DatabaseMetadata { } } -// The expected output from "codeql resolve queries" in getRemoteQueryPackDefaultQuery +// The expected output from "codeql resolve queries" in getQueryPackQueries export type ResolvedQueries = string[]; /** @@ -510,7 +510,7 @@ export type ResolvedQueries = string[]; * @param queryPack The path to the query pack on disk. * @returns The path to a query file. */ -export async function getRemoteQueryPackQueries( +export async function getQueryPackQueries( codeql: string, queryPack: string, ): Promise { From ee21911284fbb29b40b1cf42eb26d5e0d1f9429e Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Wed, 10 Jan 2024 13:55:24 +0000 Subject: [PATCH 3/5] Remove unnecessary dir check --- dist/query.js | 4 +--- src/codeql.ts | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/dist/query.js b/dist/query.js index 469587cb8..540b62d6d 100644 --- a/dist/query.js +++ b/dist/query.js @@ -74863,9 +74863,7 @@ async function adjustBqrsFiles(queryPackRunResults) { async function moveBqrsFileToResultsDir(resultsBasePath, relativeBqrsFilePath) { const oldPath = import_path.default.join(resultsBasePath, relativeBqrsFilePath); const newPath = import_path.default.join("results", relativeBqrsFilePath); - if (!import_fs2.default.existsSync(import_path.default.dirname(newPath))) { - await import_fs2.default.promises.mkdir(import_path.default.dirname(newPath), { recursive: true }); - } + await import_fs2.default.promises.mkdir(import_path.default.dirname(newPath), { recursive: true }); await import_fs2.default.promises.rename(oldPath, newPath); return newPath; } diff --git a/src/codeql.ts b/src/codeql.ts index 46ae510c1..33104e3f1 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -160,11 +160,9 @@ async function moveBqrsFileToResultsDir( const oldPath = path.join(resultsBasePath, relativeBqrsFilePath); const newPath = path.join("results", relativeBqrsFilePath); - if (!fs.existsSync(path.dirname(newPath))) { - await fs.promises.mkdir(path.dirname(newPath), { recursive: true }); - } - + await fs.promises.mkdir(path.dirname(newPath), { recursive: true }); await fs.promises.rename(oldPath, newPath); + return newPath; } From fabb0f65955a0185d09eb5345cad47713b0b798d Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Wed, 10 Jan 2024 14:24:07 +0000 Subject: [PATCH 4/5] Stop moving BQRS files around --- dist/query.js | 38 ++++++++++++++++++-------------------- src/codeql.test.ts | 4 ++-- src/codeql.ts | 41 ++++++++++++++--------------------------- src/query.ts | 11 ++++++----- 4 files changed, 40 insertions(+), 54 deletions(-) diff --git a/dist/query.js b/dist/query.js index 540b62d6d..29abcafa0 100644 --- a/dist/query.js +++ b/dist/query.js @@ -74848,24 +74848,19 @@ async function adjustBqrsFiles(queryPackRunResults) { ); const newBqrsFilePath = import_path.default.join("results", "results.bqrs"); await import_fs2.default.promises.rename(currentBqrsFilePath, newBqrsFilePath); - return [newBqrsFilePath]; + return { basePath: "results", relativeFilePaths: [newBqrsFilePath] }; } - return await Promise.all( - queryPackRunResults.queries.map(async (query) => { - const newPath = await moveBqrsFileToResultsDir( - queryPackRunResults.resultsBasePath, - query.relativeBqrsFilePath - ); - return newPath; - }) - ); -} -async function moveBqrsFileToResultsDir(resultsBasePath, relativeBqrsFilePath) { - const oldPath = import_path.default.join(resultsBasePath, relativeBqrsFilePath); - const newPath = import_path.default.join("results", relativeBqrsFilePath); - await import_fs2.default.promises.mkdir(import_path.default.dirname(newPath), { recursive: true }); - await import_fs2.default.promises.rename(oldPath, newPath); - return newPath; + for (const q of queryPackRunResults.queries) { + console.log( + `******* ${queryPackRunResults.resultsBasePath} --> ${q.relativeBqrsFilePath}` + ); + } + return { + basePath: queryPackRunResults.resultsBasePath, + relativeFilePaths: queryPackRunResults.queries.map( + (q) => q.relativeBqrsFilePath + ) + }; } async function downloadDatabase(repoId, repoName, language, pat) { let authHeader = void 0; @@ -75191,9 +75186,12 @@ async function getArtifactContentsForUpload(runQueryResult) { const sarifFileContents = import_fs3.default.createReadStream(runQueryResult.sarifFilePath); zip.file("results.sarif", sarifFileContents); } - for (const bqrsFilePath of runQueryResult.bqrsFilePaths) { - const bqrsFileContents = import_fs3.default.createReadStream(bqrsFilePath); - const relativePath = import_path2.default.relative("results", bqrsFilePath); + for (const relativePath of runQueryResult.bqrsFilePaths.relativeFilePaths) { + const fullPath = import_path2.default.join( + runQueryResult.bqrsFilePaths.basePath, + relativePath + ); + const bqrsFileContents = import_fs3.default.createReadStream(fullPath); zip.file(relativePath, bqrsFileContents); } return await zip.generateAsync({ diff --git a/src/codeql.test.ts b/src/codeql.test.ts index 5c21949c7..b07f89e78 100644 --- a/src/codeql.test.ts +++ b/src/codeql.test.ts @@ -81,7 +81,7 @@ test("running multiple queries in a pack", async (t) => { try { await runQuery("codeql", t.context.db, "a/b", queryPack); - const bqrsFilePath1 = "results/codeql/queries/x/query.bqrs"; + const bqrsFilePath1 = "db/results/codeql/queries/x/query.bqrs"; t.true(fs.existsSync(bqrsFilePath1)); const bqrsInfo1 = await getBqrsInfo("codeql", bqrsFilePath1); @@ -89,7 +89,7 @@ test("running multiple queries in a pack", async (t) => { t.is("#select", bqrsInfo1.resultSets[0].name); t.true(bqrsInfo1.compatibleQueryKinds.includes("Table")); - const bqrsFilePath2 = "results/codeql/queries/z/query.bqrs"; + const bqrsFilePath2 = "db/results/codeql/queries/z/query.bqrs"; t.true(fs.existsSync(bqrsFilePath2)); const bqrsInfo2 = await getBqrsInfo("codeql", bqrsFilePath2); diff --git a/src/codeql.ts b/src/codeql.ts index 33104e3f1..b455f5981 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -15,10 +15,15 @@ export interface RunQueryResult { databaseSHA: string | undefined; databaseName: string; sourceLocationPrefix: string; - bqrsFilePaths: string[]; + bqrsFilePaths: BqrsFilePaths; sarifFilePath?: string; } +interface BqrsFilePaths { + basePath: string; + relativeFilePaths: string[]; +} + // Must be a valid value for "-t=kind" when doing "codeql bqrs interpret" type SarifOutputType = "problem" | "path-problem"; @@ -126,7 +131,7 @@ export async function runQuery( async function adjustBqrsFiles( queryPackRunResults: QueryPackRunResults, -): Promise { +): Promise { if (queryPackRunResults.queries.length === 1) { // If we have a single query, move the BQRS file to "results.bqrs" in order to // maintain backwards compatibility with the VS Code extension, since it expects @@ -137,33 +142,15 @@ async function adjustBqrsFiles( ); const newBqrsFilePath = path.join("results", "results.bqrs"); await fs.promises.rename(currentBqrsFilePath, newBqrsFilePath); - return [newBqrsFilePath]; + return { basePath: "results", relativeFilePaths: [newBqrsFilePath] }; } - // If we have multiple queries, move the BQRS files to the correct location and - // return the new paths. - return await Promise.all( - queryPackRunResults.queries.map(async (query) => { - const newPath = await moveBqrsFileToResultsDir( - queryPackRunResults.resultsBasePath, - query.relativeBqrsFilePath, - ); - return newPath; - }), - ); -} - -async function moveBqrsFileToResultsDir( - resultsBasePath: string, - relativeBqrsFilePath: string, -): Promise { - const oldPath = path.join(resultsBasePath, relativeBqrsFilePath); - const newPath = path.join("results", relativeBqrsFilePath); - - await fs.promises.mkdir(path.dirname(newPath), { recursive: true }); - await fs.promises.rename(oldPath, newPath); - - return newPath; + return { + basePath: queryPackRunResults.resultsBasePath, + relativeFilePaths: queryPackRunResults.queries.map( + (q) => q.relativeBqrsFilePath, + ), + }; } export async function downloadDatabase( diff --git a/src/query.ts b/src/query.ts index 3e03cca63..0bf392dd3 100644 --- a/src/query.ts +++ b/src/query.ts @@ -158,11 +158,12 @@ async function getArtifactContentsForUpload( zip.file("results.sarif", sarifFileContents); } - for (const bqrsFilePath of runQueryResult.bqrsFilePaths) { - // All the files are in the results dir, so we need to make sure - // that the zip does not contain a "results" directory. - const bqrsFileContents = fs.createReadStream(bqrsFilePath); - const relativePath = path.relative("results", bqrsFilePath); + for (const relativePath of runQueryResult.bqrsFilePaths.relativeFilePaths) { + const fullPath = path.join( + runQueryResult.bqrsFilePaths.basePath, + relativePath, + ); + const bqrsFileContents = fs.createReadStream(fullPath); zip.file(relativePath, bqrsFileContents); } From 3e43d2ec727fde95b35606748f4944641a0e4dfe Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Wed, 10 Jan 2024 15:01:09 +0000 Subject: [PATCH 5/5] Add logging --- dist/query.js | 8 +++----- package.json | 4 ++-- src/query.ts | 3 +++ 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/dist/query.js b/dist/query.js index 29abcafa0..757916b68 100644 --- a/dist/query.js +++ b/dist/query.js @@ -74850,11 +74850,6 @@ async function adjustBqrsFiles(queryPackRunResults) { await import_fs2.default.promises.rename(currentBqrsFilePath, newBqrsFilePath); return { basePath: "results", relativeFilePaths: [newBqrsFilePath] }; } - for (const q of queryPackRunResults.queries) { - console.log( - `******* ${queryPackRunResults.resultsBasePath} --> ${q.relativeBqrsFilePath}` - ); - } return { basePath: queryPackRunResults.resultsBasePath, relativeFilePaths: queryPackRunResults.queries.map( @@ -75187,6 +75182,9 @@ async function getArtifactContentsForUpload(runQueryResult) { zip.file("results.sarif", sarifFileContents); } for (const relativePath of runQueryResult.bqrsFilePaths.relativeFilePaths) { + console.log( + `--------- Adding ${relativePath} to artifact. Base path is ${runQueryResult.bqrsFilePaths.basePath}` + ); const fullPath = import_path2.default.join( runQueryResult.bqrsFilePaths.basePath, relativePath diff --git a/package.json b/package.json index b6e269ada..3e5158662 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build": "node build.mjs", "watch": "node build.mjs --watch", - "test": "ava src/** --serial --verbose --timeout=1m", + "test": "ava src/codeql.test.ts --serial --verbose --timeout=1m", "test-debug": "ava src/** --serial --verbose --timeout=20m", "lint": "eslint --report-unused-disable-directives --max-warnings=0 . --ext .js,.ts", "lint-fix": "eslint --report-unused-disable-directives --max-warnings=0 . --ext .js,.ts --fix", @@ -71,4 +71,4 @@ "glob-parent": ">=5.1.2", "normalize-url": ">=4.5.1" } -} \ No newline at end of file +} diff --git a/src/query.ts b/src/query.ts index 0bf392dd3..1e9dab5cd 100644 --- a/src/query.ts +++ b/src/query.ts @@ -159,6 +159,9 @@ async function getArtifactContentsForUpload( } for (const relativePath of runQueryResult.bqrsFilePaths.relativeFilePaths) { + console.log( + `--------- Adding ${relativePath} to artifact. Base path is ${runQueryResult.bqrsFilePaths.basePath}`, + ); const fullPath = path.join( runQueryResult.bqrsFilePaths.basePath, relativePath,