package extractor

import (

"crypto/md5"

"encoding/hex"

"fmt"

"go/ast"

"go/constant"

"go/scanner"

"go/token"

"go/types"

"io"

"log"

"log/slog"

"os"

"path/filepath"

"reflect"

"regexp"

"runtime"

"strconv"

"strings"

"sync"

"time"

"github.com/github/codeql-go/extractor/dbscheme"

"github.com/github/codeql-go/extractor/diagnostics"

"github.com/github/codeql-go/extractor/srcarchive"

"github.com/github/codeql-go/extractor/toolchain"

"github.com/github/codeql-go/extractor/trap"

"github.com/github/codeql-go/extractor/util"

"golang.org/x/tools/go/packages"

)

var MaxGoRoutines int

type typeParamParentEntry struct {

parent types.Object

isFromReceiver bool

}

var typeParamParent map[*types.TypeParam]typeParamParentEntry = make(map[*types.TypeParam]typeParamParentEntry)

func init() {

// this sets the number of threads that the Go runtime will spawn; this is separate

// from the number of goroutines that the program spawns, which are scheduled into

// the system threads by the Go runtime scheduler

threads := os.Getenv("LGTM_THREADS")

if maxprocs, err := strconv.Atoi(threads); err == nil && maxprocs > 0 {

log.Printf("Max threads set to %d", maxprocs)

runtime.GOMAXPROCS(maxprocs)

} else if threads != "" {

log.Printf("Warning: LGTM_THREADS value %s is not valid, defaulting to using all available threads.", threads)

}

// if the value is empty or not set, use the Go default, which is the number of cores

// available since Go 1.5, but is subject to change

var err error

if MaxGoRoutines, err = strconv.Atoi(util.Getenv(

"CODEQL_EXTRACTOR_GO_MAX_GOROUTINES",

"SEMMLE_MAX_GOROUTINES",

)); err != nil {

MaxGoRoutines = 32

} else {

log.Printf("Max goroutines set to %d", MaxGoRoutines)

}

}

// isExactTestPackage checks if a package ID represents an exact test match.

// Returns true for IDs like "github.com/foo/bar [github.com/foo/bar.test]"

// Returns false for IDs like "github.com/foo/bar [github.com/foo/bar/nested.test]"

func isExactTestPackage(pkg *packages.Package) bool {

// Test packages have IDs in the format: "pkgpath [pkgpath.test]"

// or for nested test dependencies: "pkgpath [pkgpath/nested.test]"

expectedTestID := pkg.PkgPath + " [" + pkg.PkgPath + ".test]"

return pkg.ID == expectedTestID

}

// isBetterPackage determines if pkg is a better choice than current for extraction.

// Preferences:

// 1. Exact test package (e.g., "pkg [pkg.test]") over nested test dependencies

// 2. More Syntax nodes (more files to extract)

// 3. Longer ID string as tiebreaker

func isBetterPackage(pkg, current *packages.Package) bool {

pkgIsExact := isExactTestPackage(pkg)

currentIsExact := isExactTestPackage(current)

// Prefer exact test packages

if pkgIsExact != currentIsExact {

return pkgIsExact

}

// Prefer packages with more syntax nodes (more files)

pkgSyntaxCount := len(pkg.Syntax)

currentSyntaxCount := len(current.Syntax)

if pkgSyntaxCount != currentSyntaxCount {

return pkgSyntaxCount > currentSyntaxCount

}

// Fall back to string length

return len(pkg.ID) > len(current.ID)

}

// selectBestPackages builds a map from package paths to their best package variants.

// In the context of a `go test -c` compilation, we see the same package more than

// once, with IDs like "abc.com/pkgname [abc.com/pkgname.test]" to distinguish the version

// that contains and is used by test code.

// We prefer the version with the most complete test coverage, which is typically:

// 1. The exact test package (e.g., "pkg [pkg.test]") over nested test dependencies

// 2. The package with the most Syntax nodes (most files to extract)

// 3. The longest ID string as a tiebreaker

func selectBestPackages(pkgs []*packages.Package) map[string]*packages.Package {

bestPackageIds := make(map[string]*packages.Package)

packages.Visit(pkgs, nil, func(pkg *packages.Package) {

if bestSoFar, present := bestPackageIds[pkg.PkgPath]; present {

if isBetterPackage(pkg, bestSoFar) {

bestPackageIds[pkg.PkgPath] = pkg

}

} else {

bestPackageIds[pkg.PkgPath] = pkg

}

})

return bestPackageIds

}

// ExtractWithFlags extracts the packages specified by the given patterns and build flags

func ExtractWithFlags(buildFlags []string, patterns []string, extractTests bool, sourceRoot string) error {

startTime := time.Now()

extraction := NewExtraction(buildFlags, patterns, sourceRoot)

defer extraction.StatWriter.Close()

modEnabled := os.Getenv("GO111MODULE") != "off"

if !modEnabled {

log.Println("Go module mode disabled.")

}

modFlags := make([]string, 0, 1)

for _, flag := range buildFlags {

if strings.HasPrefix(flag, "-mod=") {

modFlags = append(modFlags, flag)

}

}

// If CODEQL_EXTRACTOR_GO_[OPTION_]EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;

// otherwise (the default) is to exclude them from extraction

includeVendor, oldOptionUsed := util.IsVendorDirExtractionEnabled()

if oldOptionUsed {

log.Println("Warning: obsolete option \"CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS\" was set. Use \"CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_VENDOR_DIRS\" or pass `--extractor-option extract_vendor_dirs=true` instead.")

}

modeNotifications := make([]string, 0, 2)

if extractTests {

modeNotifications = append(modeNotifications, "test extraction enabled")

}

if includeVendor {

modeNotifications = append(modeNotifications, "extracting vendor directories")

}

modeMessage := strings.Join(modeNotifications, ", ")

if modeMessage != "" {

modeMessage = " (" + modeMessage + ")"

}

log.Printf("Running packages.Load%s.", modeMessage)

// This includes test packages if either we're tracing a `go test` command,

// or if CODEQL_EXTRACTOR_GO_OPTION_EXTRACT_TESTS is set to "true".

cfg := &packages.Config{

Mode: packages.NeedName | packages.NeedFiles |

packages.NeedCompiledGoFiles |

packages.NeedImports | packages.NeedDeps |

packages.NeedTypes | packages.NeedTypesSizes |

packages.NeedTypesInfo | packages.NeedSyntax,

BuildFlags: buildFlags,

Tests: extractTests,

}

pkgs, err := packages.Load(cfg, patterns...)

if err != nil {

// the toolchain directive is only supported in Go 1.21 and above

if strings.Contains(err.Error(), "unknown directive: toolchain") {

diagnostics.EmitNewerSystemGoRequired("1.21.0")

}

return err

}

log.Println("Done running packages.Load.")

if len(pkgs) == 0 {

log.Println("No packages found.")

wd, err := os.Getwd()

if err != nil {

log.Printf("Warning: failed to get working directory: %s\n", err.Error())

} else if util.FindGoFiles(wd) {

diagnostics.EmitGoFilesFoundButNotProcessed()

}

}

log.Println("Extracting universe scope.")

extractUniverseScope()

log.Println("Done extracting universe scope.")

// a map of package path to source directory and module root directory

pkgInfos := make(map[string]toolchain.PkgInfo)

// root directories of packages that we want to extract

wantedRoots := make(map[string]bool)

if os.Getenv("CODEQL_EXTRACTOR_GO_FAST_PACKAGE_INFO") != "false" {

log.Printf("Running go list to resolve package and module directories.")

// get all packages information

pkgInfos, err = toolchain.GetPkgsInfo(patterns, true, extractTests, modFlags...)

if err != nil {

log.Fatalf("Error getting dependency package or module directories: %v.", err)

}

log.Printf("Done running go list deps: resolved %d packages.", len(pkgInfos))

}

pkgsNotFound := make([]string, 0, len(pkgs))

// Build a map from package paths to their best IDs

bestPackageIds := selectBestPackages(pkgs)

// Do a post-order traversal and extract the package scope of each package

packages.Visit(pkgs, nil, func(pkg *packages.Package) {

// Note that if test extraction is enabled, we will encounter a package twice here:

// once as the main package, and once as the test package (with a package ID like

// "abc.com/pkgname [abc.com/pkgname.test]").

//

// We will extract it both times however, because we need to visit the packages

// in the right order in order to visit used types before their users, and the

// ordering determined by packages.Visit for the main and the test package may differ.

//

// This should only cause some wasted time and not inconsistency because the names for

// objects seen in this process should be the same each time.

slog.Debug("Processing package", "package", pkg.PkgPath)

if _, ok := pkgInfos[pkg.PkgPath]; !ok {

pkgInfos[pkg.PkgPath] = toolchain.GetPkgInfo(pkg.PkgPath, modFlags...)

}

slog.Debug("Extracting types for package", "package", pkg.PkgPath)

tw, err := trap.NewWriter(pkg.PkgPath, pkg)

if err != nil {

log.Fatal(err)

}

defer tw.Close()

scope := extractPackageScope(tw, pkg)

extractObjectTypes(tw)

lbl := tw.Labeler.GlobalID(util.EscapeTrapSpecialChars(pkg.PkgPath) + ";pkg")

dbscheme.PackagesTable.Emit(tw, lbl, pkg.Name, pkg.PkgPath, scope)

if len(pkg.Errors) != 0 {

log.Printf("Warning: encountered errors extracting package `%s`:", pkg.PkgPath)

for i, err := range pkg.Errors {

errString := err.Error()

log.Printf(" %s", errString)

if strings.Contains(errString, "build constraints exclude all Go files in ") {

// `err` is a NoGoError from the package cmd/go/internal/load, which we cannot access as it is internal

diagnostics.EmitPackageDifferentOSArchitecture(pkg.PkgPath)

} else if strings.Contains(errString, "cannot find package") ||

strings.Contains(errString, "no required module provides package") {

pkgsNotFound = append(pkgsNotFound, pkg.PkgPath)

}

extraction.extractError(tw, err, lbl, i)

}

}

log.Printf("Done extracting types for package %s.", pkg.PkgPath)

})

if len(pkgsNotFound) > 0 {

diagnostics.EmitCannotFindPackages(diagnostics.DefaultWriter, pkgsNotFound)

}

for _, pkg := range pkgs {

pkgInfo, ok := pkgInfos[pkg.PkgPath]

if !ok || pkgInfo.PkgDir == "" {

log.Fatalf("Unable to get a source directory for input package %s.", pkg.PkgPath)

}

wantedRoots[pkgInfo.PkgDir] = true

if pkgInfo.ModDir != "" {

wantedRoots[pkgInfo.ModDir] = true

}

}

log.Println("Done processing dependencies.")

log.Println("Starting to extract packages.")

sep := regexp.QuoteMeta(string(filepath.Separator))

// Construct a list of directory segments to exclude from extraction, starting with ".."

excludedDirs := []string{`\.\.`}

if !includeVendor {

excludedDirs = append(excludedDirs, "vendor")

}

// If a path matches this regexp, we don't extract this package. It checks whether the path

// contains one of the `excludedDirs`.

noExtractRe := regexp.MustCompile(`.*(^|` + sep + `)(` + strings.Join(excludedDirs, "|") + `)($|` + sep + `).*`)

// extract AST information for all packages

packages.Visit(pkgs, nil, func(pkg *packages.Package) {

// If this is a variant of a package that also occurs with a better ID, skip it;

// otherwise we would extract the same file more than once including extracting the

// body of methods twice, causing database inconsistencies.

//

// We prefer the version with the most complete test coverage, prioritizing:

// 1. Exact test packages (e.g., "pkg [pkg.test]") over nested test dependencies

// 2. Packages with more Syntax nodes (more files to extract)

// 3. Longer ID strings as a tiebreaker

if pkg.ID != bestPackageIds[pkg.PkgPath].ID {

return

}

for root := range wantedRoots {

pkgInfo := pkgInfos[pkg.PkgPath]

relDir, err := filepath.Rel(root, pkgInfo.PkgDir)

if err != nil || noExtractRe.MatchString(relDir) {

// if the path can't be made relative or matches the noExtract regexp skip it

continue

}

extraction.extractPackage(pkg)

modDir := pkgInfo.ModDir

if modDir == "" {

modDir = pkgInfo.PkgDir

}

if modDir != "" {

modPath := filepath.Join(modDir, "go.mod")

if util.FileExists(modPath) {

log.Printf("Extracting %s", modPath)

start := time.Now()

err := extraction.extractGoMod(modPath)

if err != nil {

log.Printf("Failed to extract go.mod: %s", err.Error())

}

end := time.Since(start)

log.Printf("Done extracting %s (%dms)", modPath, end.Nanoseconds()/1000000)

}

}

return

}

log.Printf("Skipping dependency package %s.", pkg.PkgPath)

})

extraction.WaitGroup.Wait()

log.Println("Done extracting packages.")

t := time.Now()

elapsed := t.Sub(startTime)

dbscheme.CompilationFinishedTable.Emit(extraction.StatWriter, extraction.Label, 0.0, elapsed.Seconds())

return nil

}

type Extraction struct {

// A lock for preventing concurrent writes to maps and the stat trap writer, as they are not

// thread-safe

Lock sync.Mutex

LabelKey string

Label trap.Label

StatWriter *trap.Writer

WaitGroup sync.WaitGroup

GoroutineSem *semaphore

FdSem *semaphore

NextFileId int

FileInfo map[string]*FileInfo

SeenGoMods map[string]bool

OverlayChanges map[string]bool

}

type FileInfo struct {

Idx int

NextErr int

}

func (extraction *Extraction) SeenFile(path string) bool {

_, ok := extraction.FileInfo[path]

return ok

}

func (extraction *Extraction) GetFileInfo(path string) *FileInfo {

if fileInfo, ok := extraction.FileInfo[path]; ok {

return fileInfo

}

extraction.FileInfo[path] = &FileInfo{extraction.NextFileId, 0}

extraction.NextFileId += 1

return extraction.FileInfo[path]

}

func (extraction *Extraction) GetFileIdx(path string) int {

return extraction.GetFileInfo(path).Idx

}

func (extraction *Extraction) GetNextErr(path string) int {

finfo := extraction.GetFileInfo(path)

res := finfo.NextErr

finfo.NextErr += 1

return res

}

func NewExtraction(buildFlags []string, patterns []string, sourceRoot string) *Extraction {

hash := md5.New()

io.WriteString(hash, "go")

for _, buildFlag := range buildFlags {

io.WriteString(hash, " "+buildFlag)

}

io.WriteString(hash, " --")

for _, pattern := range patterns {

io.WriteString(hash, " "+pattern)

}

sum := hash.Sum(nil)

overlayChangeList := util.GetOverlayChanges(sourceRoot)

var overlayChanges map[string]bool

if overlayChangeList == nil {

overlayChanges = nil

} else {

overlayChanges = make(map[string]bool)

for _, changedFilePath := range overlayChangeList {

absPath, err := filepath.Abs(changedFilePath)

if err != nil {

log.Fatalf("Error resolving absolute path of overlay change %s: %s", changedFilePath, err.Error())

}

overlayChanges[absPath] = true

slog.Info("Overlay changed file", "path", absPath)

}

}

i := 0

var path string

// split compilation files into directories to avoid filling a single directory with too many files

pathFmt := fmt.Sprintf("compilations/%s/%s_%%d", hex.EncodeToString(sum[:1]), hex.EncodeToString(sum[1:]))

for {

path = fmt.Sprintf(pathFmt, i)

file, err := trap.FileFor(path)

if err != nil {

log.Fatalf("Error creating trap file: %s\n", err.Error())

}

i++

if !util.FileExists(file) {

break

}

}

statWriter, err := trap.NewWriter(path, nil)

if err != nil {

log.Fatal(err)

}

lblKey := fmt.Sprintf("%s_%d;compilation", hex.EncodeToString(sum), i)

lbl := statWriter.Labeler.GlobalID(lblKey)

wd, err := os.Getwd()

if err != nil {

log.Fatalf("Unable to determine current directory: %s\n", err.Error())

}

dbscheme.CompilationsTable.Emit(statWriter, lbl, wd)

i = 0

extractorPath, err := util.GetExtractorPath()

if err != nil {

log.Fatalf("Unable to get extractor path: %s\n", err.Error())

}

dbscheme.CompilationArgsTable.Emit(statWriter, lbl, 0, extractorPath)

i++

for _, flag := range buildFlags {

dbscheme.CompilationArgsTable.Emit(statWriter, lbl, i, flag)

i++

}

// emit a fake "--" argument to make it clear that what comes after it are patterns

dbscheme.CompilationArgsTable.Emit(statWriter, lbl, i, "--")

i++

for _, pattern := range patterns {

dbscheme.CompilationArgsTable.Emit(statWriter, lbl, i, pattern)

i++

}

return &Extraction{

LabelKey: lblKey,

Label: lbl,

StatWriter: statWriter,

// this semaphore is used to limit the number of files that are open at once;

// this is to prevent the extractor from running into issues with caps on the

// number of open files that can be held by one process

FdSem: newSemaphore(100),

// this semaphore is used to limit the number of goroutines spawned, so we

// don't run into memory issues

GoroutineSem: newSemaphore(MaxGoRoutines),

NextFileId: 0,

FileInfo: make(map[string]*FileInfo),

SeenGoMods: make(map[string]bool),

OverlayChanges: overlayChanges,

}

}

// extractUniverseScope extracts symbol table information for the universe scope

func extractUniverseScope() {

tw, err := trap.NewWriter("universe", nil)

if err != nil {

log.Fatal(err)

}

defer tw.Close()

lbl := tw.Labeler.ScopeID(types.Universe, nil)

dbscheme.ScopesTable.Emit(tw, lbl, dbscheme.UniverseScopeType.Index())

extractObjects(tw, types.Universe, lbl)

// Always extract an empty interface type

extractType(tw, types.NewInterfaceType([]*types.Func{}, []types.Type{}))

}

// extractObjects extracts all objects declared in the given scope

// For more information on objects, see:

// https://github.com/golang/example/blob/master/gotypes/README.md#objects

func extractObjects(tw *trap.Writer, scope *types.Scope, scopeLabel trap.Label) {

for _, name := range scope.Names() {

obj := scope.Lookup(name)

lbl, exists := tw.Labeler.ScopedObjectID(obj, func() trap.Label { return extractType(tw, obj.Type()) })

if !exists {

// Populate type parameter parents for functions. Note that methods

// do not appear as objects in any scope, so they have to be dealt

// with separately in extractMethods.

if funcObj, ok := obj.(*types.Func); ok {

populateTypeParamParentsFromFunction(funcObj)

}

// Populate type parameter parents for defined types and alias types.

if typeNameObj, ok := obj.(*types.TypeName); ok {

// `types.TypeName` represents a type with a name: a defined

// type, an alias type, a type parameter, or a predeclared

// type such as `int` or `error`. We can distinguish these

// using `typeNameObj.Type()`, except that we need to be

// careful with alias types because before Go 1.24 they would

// return the underlying type.

if tp, ok := typeNameObj.Type().(*types.Named); ok && !typeNameObj.IsAlias() {

populateTypeParamParents(tp.TypeParams(), obj, false)

} else if tp, ok := typeNameObj.Type().(*types.Alias); ok {

populateTypeParamParents(tp.TypeParams(), obj, false)

}

}

extractObject(tw, obj, lbl)

}

if obj.Parent() != scope {

// this can happen if a scope is embedded into another with a `.` import.

continue

}

dbscheme.ObjectScopesTable.Emit(tw, lbl, scopeLabel)

}

}

// extractMethod extracts a method `meth` and emits it to the objects table, then returns its label

func extractMethod(tw *trap.Writer, meth *types.Func) trap.Label {

// get the receiver type of the method

recvtyp := meth.Type().(*types.Signature).Recv().Type()

// ensure receiver type has been extracted

recvtyplbl := extractType(tw, recvtyp)

// if the method label does not exist, extract it

methlbl, exists := tw.Labeler.MethodID(meth, recvtyplbl)

if !exists {

// Populate type parameter parents for methods. They do not appear as

// objects in any scope, so they have to be dealt with separately here.

populateTypeParamParentsFromFunction(meth)

extractObject(tw, meth, methlbl)

}

return methlbl

}

// extractObject extracts a single object and emits it to the objects table.

// For more information on objects, see:

// https://github.com/golang/example/blob/master/gotypes/README.md#objects

func extractObject(tw *trap.Writer, obj types.Object, lbl trap.Label) {

checkObjectNotSpecialized(obj)

name := obj.Name()

isBuiltin := obj.Parent() == types.Universe

var kind int

switch obj.(type) {

case *types.PkgName:

kind = dbscheme.PkgObjectType.Index()

case *types.TypeName:

if isBuiltin {

kind = dbscheme.BuiltinTypeObjectType.Index()

} else {

kind = dbscheme.DeclTypeObjectType.Index()

}

case *types.Const:

if isBuiltin {

kind = dbscheme.BuiltinConstObjectType.Index()

} else {

kind = dbscheme.DeclConstObjectType.Index()

}

case *types.Nil:

kind = dbscheme.BuiltinConstObjectType.Index()

case *types.Var:

kind = dbscheme.DeclVarObjectType.Index()

case *types.Builtin:

kind = dbscheme.BuiltinFuncObjectType.Index()

case *types.Func:

kind = dbscheme.DeclFuncObjectType.Index()

case *types.Label:

kind = dbscheme.LabelObjectType.Index()

default:

log.Fatalf("unknown object of type %T", obj)

}

dbscheme.ObjectsTable.Emit(tw, lbl, kind, name)

// for methods, additionally extract information about the receiver

if sig, ok := obj.Type().(*types.Signature); ok {

if recv := sig.Recv(); recv != nil {

recvlbl, exists := tw.Labeler.ReceiverObjectID(recv, lbl)

if !exists {

extractObject(tw, recv, recvlbl)

}

dbscheme.MethodReceiversTable.Emit(tw, lbl, recvlbl)

}

}

}

// extractObjectTypes extracts type and receiver information for all objects

// For more information on objects, see:

// https://github.com/golang/example/blob/master/gotypes/README.md#objects

func extractObjectTypes(tw *trap.Writer) {

// calling `extractType` on a defined type will extract all methods defined

// on it, which will add new objects. Therefore we need to do this first

// before we loop over all objects and emit them.

changed := true

for changed {

changed = tw.ForEachObject(extractObjectType)

}

changed = tw.ForEachObject(emitObjectType)

if changed {

log.Printf("Warning: more objects were labeled while emitting object types")

}

}

// extractObjectType extracts type and receiver information for a given object

// For more information on objects, see:

// https://github.com/golang/example/blob/master/gotypes/README.md#objects

func extractObjectType(tw *trap.Writer, obj types.Object, lbl trap.Label) {

if tp := obj.Type(); tp != nil {

extractType(tw, tp)

}

}

// emitObjectType emits the type information for a given object

func emitObjectType(tw *trap.Writer, obj types.Object, lbl trap.Label) {

if tp := obj.Type(); tp != nil {

dbscheme.ObjectTypesTable.Emit(tw, lbl, extractType(tw, tp))

}

}

var (

// file:line:col

threePartPos = regexp.MustCompile(`^(.+):(\d+):(\d+)$`)

// file:line

twoPartPos = regexp.MustCompile(`^(.+):(\d+)$`)

)

// extractError extracts the message and location of a frontend error

func (extraction *Extraction) extractError(tw *trap.Writer, err packages.Error, pkglbl trap.Label, idx int) {

var (

lbl = tw.Labeler.FreshID()

tag = dbscheme.ErrorTags[err.Kind]

kind = dbscheme.ErrorTypes[err.Kind].Index()

pos = err.Pos

file = ""

line, col int

e error

)

if pos == "" || pos == "-" {

// extract a dummy file

wd, e := os.Getwd()

if e != nil {

wd = "."

log.Printf("Warning: failed to get working directory")

}

ewd, e := filepath.EvalSymlinks(wd)

if e != nil {

ewd = wd

log.Printf("Warning: failed to evaluate symlinks for %s", wd)

}

file = filepath.Join(ewd, "-")

extraction.extractFileInfo(tw, file, true)

} else {

var rawfile string

if parts := threePartPos.FindStringSubmatch(pos); parts != nil {

// "file:line:col"

col, e = strconv.Atoi(parts[3])

if e != nil {

log.Printf("Warning: malformed column number `%s`: %v", parts[3], e)

}

line, e = strconv.Atoi(parts[2])

if e != nil {

log.Printf("Warning: malformed line number `%s`: %v", parts[2], e)

}

rawfile = parts[1]

} else if parts := twoPartPos.FindStringSubmatch(pos); parts != nil {

// "file:line"

line, e = strconv.Atoi(parts[2])

if e != nil {

log.Printf("Warning: malformed line number `%s`: %v", parts[2], e)

}

rawfile = parts[1]

} else if pos != "" && pos != "-" {

log.Printf("Warning: malformed error position `%s`", pos)

}

afile, e := filepath.Abs(rawfile)

if e != nil {

log.Printf("Warning: failed to get absolute path for for %s", file)

afile = file

}

file, e = filepath.EvalSymlinks(afile)

if e != nil {

log.Printf("Warning: failed to evaluate symlinks for %s", afile)

file = afile

}

extraction.extractFileInfo(tw, file, false)

}

extraction.Lock.Lock()

flbl := extraction.StatWriter.Labeler.FileLabelFor(file)

diagLbl := extraction.StatWriter.Labeler.FreshID()

dbscheme.DiagnosticsTable.Emit(

extraction.StatWriter, diagLbl, 1, tag, err.Msg, err.Msg,

emitLocation(extraction.StatWriter, flbl, line, col, line, col))

dbscheme.DiagnosticForTable.Emit(extraction.StatWriter, diagLbl, extraction.Label, extraction.GetFileIdx(file), extraction.GetNextErr(file))

extraction.Lock.Unlock()

transformed := filepath.ToSlash(srcarchive.TransformPath(file))

dbscheme.ErrorsTable.Emit(tw, lbl, kind, err.Msg, pos, transformed, line, col, pkglbl, idx)

}

// extractPackage extracts AST information for all files in the given package

func (extraction *Extraction) extractPackage(pkg *packages.Package) {

for _, astFile := range pkg.Syntax {

extraction.WaitGroup.Add(1)

extraction.GoroutineSem.acquire(1)

go func(astFile *ast.File) {

err := extraction.extractFile(astFile, pkg)

if err != nil {

log.Fatal(err)

}

extraction.GoroutineSem.release(1)

extraction.WaitGroup.Done()

}(astFile)

}

}

// normalizedPath computes the normalized path (with symlinks resolved) for the given file

func normalizedPath(ast *ast.File, fset *token.FileSet) string {

file := fset.File(ast.Package).Name()

path, err := filepath.EvalSymlinks(file)

if err != nil {

return file

}

return util.ResolvePath(path)

}

// extractFile extracts AST information for the given file

func (extraction *Extraction) extractFile(ast *ast.File, pkg *packages.Package) error {

fset := pkg.Fset

if ast.Package == token.NoPos {

log.Printf("Skipping extracting a file without a 'package' declaration")

return nil

}

path := normalizedPath(ast, fset)

// If we're extracting an overlay, we want to skip extraction of files that haven't changed.

// Since some files may be outside the source directory (e.g. files preprocessed by cgo) we

// can't easily know if they have changed (or came from source files that changed), so we always

// extract a file if it's not in the package directory.

if extraction.OverlayChanges != nil &&

!extraction.OverlayChanges[path] &&

strings.HasPrefix(path+string(filepath.Separator), pkg.Dir) {

slog.Info("Skipping unchanged file in overlay extraction", "path", path)

return nil

}

extraction.FdSem.acquire(3)

log.Printf("Extracting %s", path)

start := time.Now()

defer extraction.FdSem.release(1)

tw, err := trap.NewWriter(path, pkg)

if err != nil {

extraction.FdSem.release(2)

return err

}

defer tw.Close()

err = srcarchive.Add(path)

extraction.FdSem.release(2)

if err != nil {

return err

}

extraction.extractFileInfo(tw, path, false)

extractScopes(tw, ast, pkg)

extractFileNode(tw, ast)

extractObjectTypes(tw)

extractNumLines(tw, path, ast)

end := time.Since(start)

log.Printf("Done extracting %s (%dms)", path, end.Nanoseconds()/1000000)

return nil

}

// extractFileInfo extracts file-system level information for the given file, populating

// the `files` and `containerparent` tables

func (extraction *Extraction) extractFileInfo(tw *trap.Writer, file string, isDummy bool) {

// We may visit the same file twice because `extractError` calls this function to describe files containing

// compilation errors. It is also called for user source files being extracted.

extraction.Lock.Lock()

if extraction.SeenFile(file) {

extraction.Lock.Unlock()

return

}

extraction.Lock.Unlock()

path := filepath.ToSlash(srcarchive.TransformPath(file))

components := strings.Split(path, "/")

parentPath := ""

var parentLbl trap.Label

for i, component := range components {

// displayPath is the same as rawPath except for root directories: if

// rawPath is "" then displayPath is "/"; if rawPath is "C:" then

// displayPath is "C:/".

var rawPath, displayPath string

if i == 0 {

rawPath = component

if component == "" || regexp.MustCompile(`^[A-Za-z]:$`).MatchString(component) {

// Handle linux root and Windows drive letters by appending "/"

displayPath = rawPath + "/"

} else {

displayPath = rawPath

}

} else {

rawPath = parentPath + "/" + component

displayPath = rawPath

}

if i == len(components)-1 {

lbl := tw.Labeler.FileLabelFor(file)

dbscheme.FilesTable.Emit(tw, lbl, displayPath)

dbscheme.ContainerParentTable.Emit(tw, parentLbl, lbl)

dbscheme.HasLocationTable.Emit(tw, lbl, emitLocation(tw, lbl, 0, 0, 0, 0))

extraction.Lock.Lock()

slbl := extraction.StatWriter.Labeler.FileLabelFor(file)

if !isDummy {

dbscheme.CompilationCompilingFilesTable.Emit(extraction.StatWriter, extraction.Label, extraction.GetFileIdx(file), slbl)

}

extraction.Lock.Unlock()

break

}

lbl := tw.Labeler.GlobalID(util.EscapeTrapSpecialChars(displayPath) + ";folder")

dbscheme.FoldersTable.Emit(tw, lbl, displayPath)

if i > 0 {

dbscheme.ContainerParentTable.Emit(tw, parentLbl, lbl)

}

parentPath = rawPath

parentLbl = lbl

}

}

// extractLocation emits a location entity for the given entity

func extractLocation(tw *trap.Writer, entity trap.Label, sl int, sc int, el int, ec int) {

filelbl := tw.Labeler.FileLabel()

dbscheme.HasLocationTable.Emit(tw, entity, emitLocation(tw, filelbl, sl, sc, el, ec))

}

// emitLocation emits a location entity

func emitLocation(tw *trap.Writer, filelbl trap.Label, sl int, sc int, el int, ec int) trap.Label {

locLbl := tw.Labeler.GlobalID(fmt.Sprintf("loc,{%s},%d,%d,%d,%d", filelbl, sl, sc, el, ec))

dbscheme.LocationsDefaultTable.Emit(tw, locLbl, filelbl, sl, sc, el, ec)

return locLbl

}

// extractNodeLocation extracts location information for the given node

func extractNodeLocation(tw *trap.Writer, nd ast.Node, lbl trap.Label) {

if nd == nil {

return

}

fset := tw.Package.Fset

start, end := fset.Position(nd.Pos()), fset.Position(nd.End())

extractLocation(tw, lbl, start.Line, start.Column, end.Line, end.Column-1)

}

// extractPackageScope extracts symbol table information for the given package

func extractPackageScope(tw *trap.Writer, pkg *packages.Package) trap.Label {

pkgScope := pkg.Types.Scope()

pkgScopeLabel := tw.Labeler.ScopeID(pkgScope, pkg.Types)

dbscheme.ScopesTable.Emit(tw, pkgScopeLabel, dbscheme.PackageScopeType.Index())

dbscheme.ScopeNestingTable.Emit(tw, pkgScopeLabel, tw.Labeler.ScopeID(types.Universe, nil))

extractObjects(tw, pkgScope, pkgScopeLabel)

return pkgScopeLabel

}

// extractScopeLocation extracts location information for the given scope

func extractScopeLocation(tw *trap.Writer, scope *types.Scope, lbl trap.Label) {

fset := tw.Package.Fset

start, end := fset.Position(scope.Pos()), fset.Position(scope.End())

extractLocation(tw, lbl, start.Line, start.Column, end.Line, end.Column-1)

}

// extractScopes extracts symbol table information for the package scope and all local scopes

// of the given package. Note that this will not encounter methods or struct fields as

// they do not have a parent scope.

func extractScopes(tw *trap.Writer, nd *ast.File, pkg *packages.Package) {

pkgScopeLabel := extractPackageScope(tw, pkg)

fileScope := pkg.TypesInfo.Scopes[nd]

if fileScope != nil {

extractLocalScope(tw, fileScope, pkgScopeLabel)

}

}

// extractLocalScope extracts symbol table information for the given scope and all its nested scopes

func extractLocalScope(tw *trap.Writer, scope *types.Scope, parentScopeLabel trap.Label) {

scopeLabel := tw.Labeler.ScopeID(scope, nil)

dbscheme.ScopesTable.Emit(tw, scopeLabel, dbscheme.LocalScopeType.Index())

extractScopeLocation(tw, scope, scopeLabel)

dbscheme.ScopeNestingTable.Emit(tw, scopeLabel, parentScopeLabel)

for i := 0; i < scope.NumChildren(); i++ {

childScope := scope.Child(i)

extractLocalScope(tw, childScope, scopeLabel)

}

extractObjects(tw, scope, scopeLabel)

}

// extractFileNode extracts AST information for the given file and all nodes contained in it

func extractFileNode(tw *trap.Writer, nd *ast.File) {

lbl := tw.Labeler.FileLabel()

extractExpr(tw, nd.Name, lbl, 0, false)

for i, decl := range nd.Decls {

extractDecl(tw, decl, lbl, i)

}

for i, cg := range nd.Comments {

extractCommentGroup(tw, cg, lbl, i)

}

extractDoc(tw, nd.Doc, lbl)

emitScopeNodeInfo(tw, nd, lbl)

}

// extractDoc extracts information about a doc comment group associated with a given element

func extractDoc(tw *trap.Writer, doc *ast.CommentGroup, elt trap.Label) {

if doc != nil {

dbscheme.DocCommentsTable.Emit(tw, elt, tw.Labeler.LocalID(doc))

}

}

// extractCommentGroup extracts information about a doc comment group

func extractCommentGroup(tw *trap.Writer, cg *ast.CommentGroup, parent trap.Label, idx int) {

lbl := tw.Labeler.LocalID(cg)

dbscheme.CommentGroupsTable.Emit(tw, lbl, parent, idx)

extractNodeLocation(tw, cg, lbl)

for i, c := range cg.List {

extractComment(tw, c, lbl, i)

}

}

// extractComment extracts information about a given comment

func extractComment(tw *trap.Writer, c *ast.Comment, parent trap.Label, idx int) {

lbl := tw.Labeler.LocalID(c)

rawText := c.Text

var kind int

var text string

if rawText[:2] == "//" {

kind = dbscheme.SlashSlashComment.Index()

text = rawText[2:]

} else {

kind = dbscheme.SlashStarComment.Index()

text = rawText[2 : len(rawText)-2]