From e00c7f45ef1904531eef12b4785ec02aab450678 Mon Sep 17 00:00:00 2001 From: erik-krogh Date: Mon, 24 Oct 2022 21:20:17 +0200 Subject: [PATCH] create a shared implementation for files and locations --- javascript/ql/lib/qlpack.yml | 2 + .../javascript/internal/LocationsImpl.qll | 37 +++ ruby/ql/lib/codeql/Locations.qll | 63 +--- ruby/ql/lib/codeql/files/FileSystem.qll | 178 +---------- .../codeql/ruby/internal/LocationsImpl.qll | 37 +++ ruby/ql/lib/qlpack.yml | 1 + .../2022-09-26-initial-version.md | 4 + shared/utils/codeql/utils/Locations.qll | 296 ++++++++++++++++++ shared/utils/qlpack.yml | 4 + 9 files changed, 387 insertions(+), 235 deletions(-) create mode 100644 javascript/ql/lib/semmle/javascript/internal/LocationsImpl.qll create mode 100644 ruby/ql/lib/codeql/ruby/internal/LocationsImpl.qll create mode 100644 shared/utils/change-notes/2022-09-26-initial-version.md create mode 100644 shared/utils/codeql/utils/Locations.qll create mode 100644 shared/utils/qlpack.yml diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml index 2295fcf87150..a27c045f4cf4 100644 --- a/javascript/ql/lib/qlpack.yml +++ b/javascript/ql/lib/qlpack.yml @@ -5,3 +5,5 @@ dbscheme: semmlecode.javascript.dbscheme extractor: javascript library: true upgrades: upgrades +dependencies: + codeql/utils: 0.0.1 diff --git a/javascript/ql/lib/semmle/javascript/internal/LocationsImpl.qll b/javascript/ql/lib/semmle/javascript/internal/LocationsImpl.qll new file mode 100644 index 000000000000..3e55e129ee98 --- /dev/null +++ b/javascript/ql/lib/semmle/javascript/internal/LocationsImpl.qll @@ -0,0 +1,37 @@ +/** + * Provides an implementation of `LocationsSig` from the `codeql/utils` package. + */ + +private import codeql.utils.Locations as Locs + +/** An implementation of `LocationsSig`. */ +module LocationsImpl implements Locs::LocationsSig { + abstract class Container extends @container { + abstract string getAbsolutePath(); + + string toString() { result = this.getAbsolutePath() } + + Container getParentContainer() { containerparent(result, this) } + } + + class File extends @file, Container { + override string getAbsolutePath() { files(this, result) } + } + + class Folder extends @folder, Container { + override string getAbsolutePath() { folders(this, result) } + } + + string getSourceLocationPrefix() { sourceLocationPrefix(result) } + + class Location = @location; + + predicate locations( + Location loc, File file, int startLine, int startColum, int endLine, int endColumn + ) { + locations_default(loc, file, startLine, startColum, endLine, endColumn) + } +} + +/** An instantiation of the shared Locations module. */ +module Inst = Locs::Make; diff --git a/ruby/ql/lib/codeql/Locations.qll b/ruby/ql/lib/codeql/Locations.qll index 8bed8d904f16..2d4b0c147c6c 100644 --- a/ruby/ql/lib/codeql/Locations.qll +++ b/ruby/ql/lib/codeql/Locations.qll @@ -1,64 +1,5 @@ -/** Provides classes for working with locations. */ - -import files.FileSystem - -/** - * A location as given by a file, a start line, a start column, - * an end line, and an end column. - * - * For more information about locations see [LGTM locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). - */ -class Location extends @location { - /** Gets the file for this location. */ - File getFile() { locations_default(this, result, _, _, _, _) } - - /** Gets the 1-based line number (inclusive) where this location starts. */ - int getStartLine() { locations_default(this, _, result, _, _, _) } - - /** Gets the 1-based column number (inclusive) where this location starts. */ - int getStartColumn() { locations_default(this, _, _, result, _, _) } - - /** Gets the 1-based line number (inclusive) where this location ends. */ - int getEndLine() { locations_default(this, _, _, _, result, _) } - - /** Gets the 1-based column number (inclusive) where this location ends. */ - int getEndColumn() { locations_default(this, _, _, _, _, result) } - - /** Gets the number of lines covered by this location. */ - int getNumLines() { result = this.getEndLine() - this.getStartLine() + 1 } - - /** Gets a textual representation of this element. */ - string toString() { - exists(string filepath, int startline, int startcolumn, int endline, int endcolumn | - this.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and - result = filepath + "@" + startline + ":" + startcolumn + ":" + endline + ":" + endcolumn - ) - } - - /** - * Holds if this element is at the specified location. - * The location spans column `startcolumn` of line `startline` to - * column `endcolumn` of line `endline` in file `filepath`. - * For more information, see - * [LGTM locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). - */ - predicate hasLocationInfo( - string filepath, int startline, int startcolumn, int endline, int endcolumn - ) { - exists(File f | - locations_default(this, f, startline, startcolumn, endline, endcolumn) and - filepath = f.getAbsolutePath() - ) - } - - /** Holds if this location starts strictly before the specified location. */ - pragma[inline] - predicate strictlyBefore(Location other) { - this.getStartLine() < other.getStartLine() - or - this.getStartLine() = other.getStartLine() and this.getStartColumn() < other.getStartColumn() - } -} +private import codeql.utils.Locations as Locs +import codeql.ruby.internal.LocationsImpl::Inst /** An entity representing an empty location. */ class EmptyLocation extends Location { diff --git a/ruby/ql/lib/codeql/files/FileSystem.qll b/ruby/ql/lib/codeql/files/FileSystem.qll index 552b85a4673f..99da83b266fd 100644 --- a/ruby/ql/lib/codeql/files/FileSystem.qll +++ b/ruby/ql/lib/codeql/files/FileSystem.qll @@ -1,177 +1,7 @@ -/** Provides classes for working with files and folders. */ +private import codeql.ruby.internal.LocationsImpl -private import codeql.Locations +class Container = Inst::Container; -/** A file or folder. */ -abstract class Container extends @container { - /** Gets a file or sub-folder in this container. */ - Container getAChildContainer() { this = result.getParentContainer() } +class File = Inst::File; - /** Gets a file in this container. */ - File getAFile() { result = this.getAChildContainer() } - - /** Gets a sub-folder in this container. */ - Folder getAFolder() { result = this.getAChildContainer() } - - /** - * Gets the absolute, canonical path of this container, using forward slashes - * as path separator. - * - * The path starts with a _root prefix_ followed by zero or more _path - * segments_ separated by forward slashes. - * - * The root prefix is of one of the following forms: - * - * 1. A single forward slash `/` (Unix-style) - * 2. An upper-case drive letter followed by a colon and a forward slash, - * such as `C:/` (Windows-style) - * 3. Two forward slashes, a computer name, and then another forward slash, - * such as `//FileServer/` (UNC-style) - * - * Path segments are never empty (that is, absolute paths never contain two - * contiguous slashes, except as part of a UNC-style root prefix). Also, path - * segments never contain forward slashes, and no path segment is of the - * form `.` (one dot) or `..` (two dots). - * - * Note that an absolute path never ends with a forward slash, except if it is - * a bare root prefix, that is, the path has no path segments. A container - * whose absolute path has no segments is always a `Folder`, not a `File`. - */ - abstract string getAbsolutePath(); - - /** - * Gets the base name of this container including extension, that is, the last - * segment of its absolute path, or the empty string if it has no segments. - * - * Here are some examples of absolute paths and the corresponding base names - * (surrounded with quotes to avoid ambiguity): - * - * - * - * - * - * - * - * - * - *
Absolute pathBase name
"/tmp/tst.go""tst.go"
"C:/Program Files (x86)""Program Files (x86)"
"/"""
"C:/"""
"D:/"""
"//FileServer/"""
- */ - string getBaseName() { - result = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1) - } - - /** - * Gets the extension of this container, that is, the suffix of its base name - * after the last dot character, if any. - * - * In particular, - * - * - if the name does not include a dot, there is no extension, so this - * predicate has no result; - * - if the name ends in a dot, the extension is the empty string; - * - if the name contains multiple dots, the extension follows the last dot. - * - * Here are some examples of absolute paths and the corresponding extensions - * (surrounded with quotes to avoid ambiguity): - * - * - * - * - * - * - * - * - *
Absolute pathExtension
"/tmp/tst.go""go"
"/tmp/.classpath""classpath"
"/bin/bash"not defined
"/tmp/tst2."""
"/tmp/x.tar.gz""gz"
- */ - string getExtension() { - result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3) - } - - /** Gets the file in this container that has the given `baseName`, if any. */ - File getFile(string baseName) { - result = this.getAFile() and - result.getBaseName() = baseName - } - - /** Gets the sub-folder in this container that has the given `baseName`, if any. */ - Folder getFolder(string baseName) { - result = this.getAFolder() and - result.getBaseName() = baseName - } - - /** Gets the parent container of this file or folder, if any. */ - Container getParentContainer() { containerparent(result, this) } - - /** - * Gets the relative path of this file or folder from the root folder of the - * analyzed source location. The relative path of the root folder itself is - * the empty string. - * - * This has no result if the container is outside the source root, that is, - * if the root folder is not a reflexive, transitive parent of this container. - */ - string getRelativePath() { - exists(string absPath, string pref | - absPath = this.getAbsolutePath() and sourceLocationPrefix(pref) - | - absPath = pref and result = "" - or - absPath = pref.regexpReplaceAll("/$", "") + "/" + result and - not result.matches("/%") - ) - } - - /** - * Gets the stem of this container, that is, the prefix of its base name up to - * (but not including) the last dot character if there is one, or the entire - * base name if there is not. - * - * Here are some examples of absolute paths and the corresponding stems - * (surrounded with quotes to avoid ambiguity): - * - * - * - * - * - * - * - * - *
Absolute pathStem
"/tmp/tst.go""tst"
"/tmp/.classpath"""
"/bin/bash""bash"
"/tmp/tst2.""tst2"
"/tmp/x.tar.gz""x.tar"
- */ - string getStem() { - result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1) - } - - /** - * Gets a URL representing the location of this container. - * - * For more information see https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-urls. - */ - abstract string getURL(); - - /** - * Gets a textual representation of the path of this container. - * - * This is the absolute path of the container. - */ - string toString() { result = this.getAbsolutePath() } -} - -/** A folder. */ -class Folder extends Container, @folder { - override string getAbsolutePath() { folders(this, result) } - - /** Gets the URL of this folder. */ - override string getURL() { result = "folder://" + this.getAbsolutePath() } -} - -/** A file. */ -class File extends Container, @file { - override string getAbsolutePath() { files(this, result) } - - /** Gets the URL of this file. */ - override string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" } - - /** Holds if this file was extracted from ordinary source code. */ - predicate fromSource() { any() } -} +class Folder = Inst::Folder; diff --git a/ruby/ql/lib/codeql/ruby/internal/LocationsImpl.qll b/ruby/ql/lib/codeql/ruby/internal/LocationsImpl.qll new file mode 100644 index 000000000000..c5446f1156e0 --- /dev/null +++ b/ruby/ql/lib/codeql/ruby/internal/LocationsImpl.qll @@ -0,0 +1,37 @@ +/** + * Provides an implementation of `LocationsSig` from the `codeql/utils` package. + */ + +private import codeql.utils.Locations as Locs + +/** An implementation of `LocationsSig`. */ +module LocationsImpl implements Locs::LocationsSig { + abstract class Container extends @container { + abstract string getAbsolutePath(); + + Container getParentContainer() { containerparent(result, this) } + + string toString() { result = this.getAbsolutePath() } + } + + class File extends @file, Container { + override string getAbsolutePath() { files(this, result) } + } + + class Folder extends Container, @folder { + override string getAbsolutePath() { folders(this, result) } + } + + string getSourceLocationPrefix() { sourceLocationPrefix(result) } + + class Location = @location_default; + + predicate locations( + Location loc, File file, int startLine, int startColum, int endLine, int endColumn + ) { + locations_default(loc, file, startLine, startColum, endLine, endColumn) + } +} + +/** An instantiation of the shared Locations module. */ +module Inst = Locs::Make; diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index f1fccb7524fd..dc4bc38e4864 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -7,3 +7,4 @@ upgrades: upgrades library: true dependencies: codeql/ssa: 0.0.1 + codeql/utils: 0.0.1 diff --git a/shared/utils/change-notes/2022-09-26-initial-version.md b/shared/utils/change-notes/2022-09-26-initial-version.md new file mode 100644 index 000000000000..52ee507caaac --- /dev/null +++ b/shared/utils/change-notes/2022-09-26-initial-version.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Initial release. Extracted common file system and locations code into a library pack to share code between languages. diff --git a/shared/utils/codeql/utils/Locations.qll b/shared/utils/codeql/utils/Locations.qll new file mode 100644 index 000000000000..855583a5bc11 --- /dev/null +++ b/shared/utils/codeql/utils/Locations.qll @@ -0,0 +1,296 @@ +/** + * A shared library for reasoning about locations. + * - `LocationsSig` is a basic API that every language is expected to implement. + * - `Make` is a full implementation of a Location API. + */ + +/** Provides classes for working with locations. */ +signature module LocationsSig { + class Location; + + predicate locations( + Location loc, File file, int startLine, int startColum, int endLine, int endColumn + ); + + class Container { + string getAbsolutePath(); + + Container getParentContainer(); + } + + class File extends Container; + + class Folder extends Container; + + string getSourceLocationPrefix(); +} + +/** A API for working with Locations. */ +module Make { + /** A file or folder. */ + abstract class Container instanceof LocImpl::Container { + /** Gets a file or sub-folder in this container. */ + Container getAChildContainer() { this = result.getParentContainer() } + + /** Gets a file in this container. */ + File getAFile() { result = this.getAChildContainer() } + + /** Gets a sub-folder in this container. */ + Folder getAFolder() { result = this.getAChildContainer() } + + /** + * Gets the absolute, canonical path of this container, using forward slashes + * as path separator. + * + * The path starts with a _root prefix_ followed by zero or more _path + * segments_ separated by forward slashes. + * + * The root prefix is of one of the following forms: + * + * 1. A single forward slash `/` (Unix-style) + * 2. An upper-case drive letter followed by a colon and a forward slash, + * such as `C:/` (Windows-style) + * 3. Two forward slashes, a computer name, and then another forward slash, + * such as `//FileServer/` (UNC-style) + * + * Path segments are never empty (that is, absolute paths never contain two + * contiguous slashes, except as part of a UNC-style root prefix). Also, path + * segments never contain forward slashes, and no path segment is of the + * form `.` (one dot) or `..` (two dots). + * + * Note that an absolute path never ends with a forward slash, except if it is + * a bare root prefix, that is, the path has no path segments. A container + * whose absolute path has no segments is always a `Folder`, not a `File`. + */ + string getAbsolutePath() { result = LocImpl::Container.super.getAbsolutePath() } + + /** + * Gets the base name of this container including extension, that is, the last + * segment of its absolute path, or the empty string if it has no segments. + * + * Here are some examples of absolute paths and the corresponding base names + * (surrounded with quotes to avoid ambiguity): + * + * + * + * + * + * + * + * + * + *
Absolute pathBase name
"/tmp/tst.go""tst.go"
"C:/Program Files (x86)""Program Files (x86)"
"/"""
"C:/"""
"D:/"""
"//FileServer/"""
+ */ + string getBaseName() { + result = this.getAbsolutePath().regexpCapture(".*/(([^/]*?)(?:\\.([^.]*))?)", 1) + } + + /** + * Gets the extension of this container, that is, the suffix of its base name + * after the last dot character, if any. + * + * In particular, + * + * - if the name does not include a dot, there is no extension, so this + * predicate has no result; + * - if the name ends in a dot, the extension is the empty string; + * - if the name contains multiple dots, the extension follows the last dot. + * + * Here are some examples of absolute paths and the corresponding extensions + * (surrounded with quotes to avoid ambiguity): + * + * + * + * + * + * + * + * + *
Absolute pathExtension
"/tmp/tst.go""go"
"/tmp/.classpath""classpath"
"/bin/bash"not defined
"/tmp/tst2."""
"/tmp/x.tar.gz""gz"
+ */ + string getExtension() { + result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(\\.([^.]*))?", 3) + } + + /** Gets the file in this container that has the given `baseName`, if any. */ + File getFile(string baseName) { + result = this.getAFile() and + result.getBaseName() = baseName + } + + /** Gets the sub-folder in this container that has the given `baseName`, if any. */ + Folder getFolder(string baseName) { + result = this.getAFolder() and + result.getBaseName() = baseName + } + + /** Gets the parent container of this file or folder, if any. */ + Container getParentContainer() { result = LocImpl::Container.super.getParentContainer() } + + /** + * Gets the relative path of this file or folder from the root folder of the + * analyzed source location. The relative path of the root folder itself is + * the empty string. + * + * This has no result if the container is outside the source root, that is, + * if the root folder is not a reflexive, transitive parent of this container. + */ + string getRelativePath() { + exists(string absPath, string pref | + absPath = this.getAbsolutePath() and pref = LocImpl::getSourceLocationPrefix() + | + absPath = pref and result = "" + or + absPath = pref.regexpReplaceAll("/$", "") + "/" + result and + not result.matches("/%") + ) + } + + /** + * Gets the stem of this container, that is, the prefix of its base name up to + * (but not including) the last dot character if there is one, or the entire + * base name if there is not. + * + * Here are some examples of absolute paths and the corresponding stems + * (surrounded with quotes to avoid ambiguity): + * + * + * + * + * + * + * + * + *
Absolute pathStem
"/tmp/tst.go""tst"
"/tmp/.classpath"""
"/bin/bash""bash"
"/tmp/tst2.""tst2"
"/tmp/x.tar.gz""x.tar"
+ */ + string getStem() { + result = this.getAbsolutePath().regexpCapture(".*/([^/]*?)(?:\\.([^.]*))?", 1) + } + + /** + * Gets a URL representing the location of this container. + * + * For more information see https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/#providing-urls. + */ + abstract string getURL(); + + /** + * Gets a textual representation of the path of this container. + * + * This is the absolute path of the container. + */ + string toString() { result = this.getAbsolutePath() } + } + + /** A folder. */ + class Folder extends Container instanceof LocImpl::Folder { + /** Gets the URL of this folder. */ + override string getURL() { result = "folder://" + this.getAbsolutePath() } + + /** Gets the file or subfolder in this folder that has the given `name`, if any. */ + Container getChildContainer(string name) { + result = this.getAChildContainer() and + result.getBaseName() = name + } + + /** Gets the file in this folder that has the given `stem` and `extension`, if any. */ + File getFile(string stem, string extension) { + result = this.getAChildContainer() and + result.getStem() = stem and + result.getExtension() = extension + } + + /** Gets a subfolder contained in this folder. */ + Folder getASubFolder() { result = this.getAChildContainer() } + } + + /** A file. */ + class File extends Container instanceof LocImpl::File { + /** Gets the URL of this file. */ + override string getURL() { result = "file://" + this.getAbsolutePath() + ":0:0:0:0" } + + /** Holds if this file was extracted from ordinary source code. */ + predicate fromSource() { any() } + } + + /** + * A location as given by a file, a start line, a start column, + * an end line, and an end column. + * + * For more information about locations see [LGTM locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ + class Location instanceof LocImpl::Location { + /** Gets the file for this location. */ + File getFile() { LocImpl::locations(this, result, _, _, _, _) } + + /** Gets the 1-based line number (inclusive) where this location starts. */ + int getStartLine() { LocImpl::locations(this, _, result, _, _, _) } + + /** Gets the 1-based column number (inclusive) where this location starts. */ + int getStartColumn() { LocImpl::locations(this, _, _, result, _, _) } + + /** Gets the 1-based line number (inclusive) where this location ends. */ + int getEndLine() { LocImpl::locations(this, _, _, _, result, _) } + + /** Gets the 1-based column number (inclusive) where this location ends. */ + int getEndColumn() { LocImpl::locations(this, _, _, _, _, result) } + + /** Gets the number of lines covered by this location. */ + int getNumLines() { result = this.getEndLine() - this.getStartLine() + 1 } + + /** Gets a textual representation of this element. */ + string toString() { + exists(string filepath, int startline, int startcolumn, int endline, int endcolumn | + this.hasLocationInfo(filepath, startline, startcolumn, endline, endcolumn) and + result = filepath + "@" + startline + ":" + startcolumn + ":" + endline + ":" + endcolumn + ) + } + + /** + * Holds if this element is at the specified location. + * The location spans column `startcolumn` of line `startline` to + * column `endcolumn` of line `endline` in file `filepath`. + * For more information, see + * [LGTM locations](https://codeql.github.com/docs/writing-codeql-queries/providing-locations-in-codeql-queries/). + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + exists(File f | + LocImpl::locations(this, f, startline, startcolumn, endline, endcolumn) and + filepath = f.getAbsolutePath() + ) + } + + /** Holds if this location starts strictly before the specified location. */ + pragma[inline] + predicate strictlyBefore(Location that) { + this.getFile() = that.getFile() and + ( + this.getStartLine() < that.getStartLine() + or + this.getStartLine() = that.getStartLine() and this.getStartColumn() < that.getStartColumn() + ) + } + + /** Holds if this location ends after location `that`. */ + pragma[inline] + predicate endsAfter(Location that) { + this.getFile() = that.getFile() and + ( + this.getEndLine() > that.getEndLine() + or + this.getEndLine() = that.getEndLine() and this.getEndColumn() > that.getEndColumn() + ) + } + + /** + * Holds if this location contains location `that`, meaning that it starts + * before and ends after it. + */ + predicate contains(Location that) { this.strictlyBefore(that) and this.endsAfter(that) } + + /** Holds if this location is empty. */ + predicate isEmpty() { exists(int l, int c | LocImpl::locations(this, _, l, c, l, c - 1)) } + } +} diff --git a/shared/utils/qlpack.yml b/shared/utils/qlpack.yml new file mode 100644 index 000000000000..9cae4675e880 --- /dev/null +++ b/shared/utils/qlpack.yml @@ -0,0 +1,4 @@ +name: codeql/utils +version: 0.0.1-dev +groups: shared +library: true