ignore automated test files in linter (github#35632)

rachmari · web-flow · commit 50f166fecbd7 · 2023-03-16T22:00:46.000Z
diff --git a/src/codeql-cli/lib/config.json b/src/codeql-cli/lib/config.json
@@ -18,5 +18,6 @@
   },
   "targetDirectory": "content/code-security/code-scanning/codeql-cli-manual",
   "sourceDirectory": "semmle-code/documentation/restructuredtext/codeql-cli/manual",
-  "removeKeywords": ["[Plumbing]"]
+  "removeKeywords": ["[Plumbing]"],
+  "linterIgnore": ["content/code-security/code-scanning/codeql-cli-manual"]
 }
diff --git a/tests/linting/lint-files.js b/tests/linting/lint-files.js
@@ -10,6 +10,7 @@ import addFormats from 'ajv-formats'
 import { fromMarkdown } from 'mdast-util-from-markdown'
 import { visit } from 'unist-util-visit'
 import fs from 'fs/promises'
+import { existsSync } from 'fs'
 import semver from 'semver'
 import { frontmatter, deprecatedProperties } from '../../lib/frontmatter.js'
 import languages from '../../lib/languages.js'
@@ -236,7 +237,53 @@ let mdToLint, ymlToLint, ghesReleaseNotesToLint, ghaeReleaseNotesToLint, learnin
 
 const contentMarkdownAbsPaths = walk(contentDir, mdWalkOptions).sort()
 const contentMarkdownRelPaths = contentMarkdownAbsPaths.map((p) => slash(path.relative(rootDir, p)))
-const contentMarkdownTuples = zip(contentMarkdownRelPaths, contentMarkdownAbsPaths)
+
+// Get the list of config files for automated pipelines
+const automatedConfigFiles = walk(`src`, { includeBasePath: true, globs: ['**/lib/config.json'] })
+// Get a list of Markdown files to ignore during Markdown linting
+const automatedIgnorePaths = (
+  await Promise.all(
+    automatedConfigFiles.map(async (p) => {
+      return JSON.parse(await fs.readFile(p, 'utf8')).linterIgnore
+    })
+  )
+)
+  .flat()
+  .filter(Boolean)
+console.log(automatedIgnorePaths)
+// For each linterIgnore directory, walk the files in the directory and add
+// to the ignore list.
+const ignoreMarkdownFilesAbsPath = new Set(
+  automatedIgnorePaths
+    .filter((p) => {
+      const exists = existsSync(p)
+      if (!exists) {
+        console.warn(
+          `WARNING: Ignored path ${p} defined in an automation pipeline does not exist. This may be expected, but if not, remove the defined path from the pipeline config.`
+        )
+      }
+      return exists
+    })
+    .map((p) =>
+      walk(p, {
+        includeBasePath: true,
+        globs: ['**/*.md'],
+      })
+    )
+    .flat()
+)
+
+// Difference between contentMarkdownAbsPaths & automatedIgnorePaths
+const contentMarkdownNoAutomated = [...contentMarkdownRelPaths].filter(
+  (p) => !ignoreMarkdownFilesAbsPath.has(p)
+)
+// We also need to go back and get the difference between the
+// absolute paths list
+const contentMarkdownAbsPathNoAutomated = [...contentMarkdownAbsPaths].filter(
+  (p) => !ignoreMarkdownFilesAbsPath.has(slash(path.relative(rootDir, p)))
+)
+
+const contentMarkdownTuples = zip(contentMarkdownNoAutomated, contentMarkdownAbsPathNoAutomated)
 
 const reusableMarkdownAbsPaths = walk(reusablesDir, mdWalkOptions).sort()
 const reusableMarkdownRelPaths = reusableMarkdownAbsPaths.map((p) =>

Original file line number	Diff line number	Diff line change
`@@ -18,5 +18,6 @@`
`18`	`18`	`},`
`19`	`19`	`"targetDirectory": "content/code-security/code-scanning/codeql-cli-manual",`
`20`	`20`	`"sourceDirectory": "semmle-code/documentation/restructuredtext/codeql-cli/manual",`
`21`		`- "removeKeywords": ["[Plumbing]"]`
	`21`	`+ "removeKeywords": ["[Plumbing]"],`
	`22`	`+ "linterIgnore": ["content/code-security/code-scanning/codeql-cli-manual"]`
`22`	`23`	`}`