Run CI only for tasks affected by git changes

DataDog · smola · Aug 21, 2024 · Aug 12, 2024 · Aug 19, 2024 · Aug 19, 2024
commit ee5dc89b7461434f9659f73bab49adc988b009ba
@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;
@@ -32,7 +32,7 @@ system_test_matrix: &system_test_matrix
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
-instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
+instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -312,6 +317,9 @@ jobs:
             ./gradlew clean
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
             --rerun-tasks
@@ -411,6 +419,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             << pipeline.parameters.gradle_flags >>
@@ -556,6 +567,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>

@@ -52,7 +52,7 @@
             )
             resp.raise_for_status()
         except Exception as e:
-            print(f"Request filed: {e}")
+            print(f"Request failed: {e}")
             time.sleep(1)
             continue
         data = resp.json()
@@ -63,12 +63,18 @@
     labels = {
         l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
     }
+    # get the base reference (e.g. `master`), commit hash is also available at the `sha` field.
+    pr_base_ref = data.get("base", {}).get("ref")
 else:
     labels = set()
+    pr_base_ref = ""
 
 
 branch = os.environ.get("CIRCLE_BRANCH", "")
-if branch == "master" or branch.startswith("release/v") or "all" in labels:
+run_all = "all" in labels
+is_master_or_release = branch == "master" or branch.startswith("release/v")
+
+if is_master_or_release or run_all:
     all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
 else:
     all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
@@ -83,6 +89,9 @@
 is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true"
 is_regular = not is_nightly and not is_weekly
 
+# Use git changes detection on PRs
+use_git_changes = not run_all and not is_master_or_release and is_regular
+
 vars = {
     "is_nightly": is_nightly,
     "is_weekly": is_weekly,
@@ -92,12 +101,14 @@
     "nocov_jdks": nocov_jdks,
     "flaky": branch == "master" or "flaky" in labels or "all" in labels,
     "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-",
+    "use_git_changes": use_git_changes,
+    "pr_base_ref": pr_base_ref,
 }
 
 print(f"Variables for this build: {vars}")
 
 loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
-env = jinja2.Environment(loader=loader)
+env = jinja2.Environment(loader=loader, trim_blocks=True)
 tpl = env.get_template(TPL_FILENAME)
 out = tpl.render(**vars)
 

@@ -1,3 +1,5 @@
+import java.nio.file.Paths
+
 buildscript {
   dependencies {
     classpath "pl.allegro.tech.build:axion-release-plugin:1.14.4"
@@ -164,18 +166,169 @@ allprojects { project ->
   }
 }
 
+Set<Task> getTaskDependenciesRecursive(Task baseTask, Set<Task> visited = []) {
+  if (visited.contains(baseTask)) {
+    return []
+  }
+  Set<Task> dependencies = [baseTask]
+  visited.add(baseTask)
+  for (td in baseTask.taskDependencies) {
+    for (t in td.getDependencies(baseTask)) {
+      dependencies.add(t)
+      dependencies.addAll(getTaskDependenciesRecursive(t, visited))
+    }
+  }
+  return dependencies
+}
+
+File relativeToGitRoot(File f) {
+  return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile()
+}
+
+String isAffectedBy(Task baseTask, Map<Project, Set<String>> affectedProjects) {
+  for (Task t in getTaskDependenciesRecursive(baseTask)) {
+    if (!affectedProjects.containsKey(t.project)) {
+      continue
+    }
+    final Set<String> affectedTasks = affectedProjects.get(t.project)
+    if (affectedTasks.contains("all")) {
+      return "${t.project.path}:${t.name}"
+    }
+    if (affectedTasks.contains(t.name)) {
+      return "${t.project.path}:${t.name}"
+    }
+  }
+  return null
+}
+
+List<File> getChangedFiles(String baseRef, String newRef) {
+  final stdout = new StringBuilder()
+  final stderr = new StringBuilder()
+  final proc = "git diff --name-only ${baseRef}..${newRef}".execute()
+  proc.consumeProcessOutput(stdout, stderr)
+  proc.waitForOrKill(1000)
+  assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}"
+  def out = stdout.toString().trim()
+  if (out.isEmpty()) {
+    return []
+  }
+  logger.debug("git diff output: ${out}")
+  return out.split("\n").collect {
+    new File(rootProject.projectDir, it.trim())
+  }
+}
+
+rootProject.ext {
+  useGitChanges = false
+}
+
+if (rootProject.hasProperty("gitBaseRef")) {
+  // -PgitBaseRef sets the base git reference to compare changes to. In CI, this should generally be set to the target
+  // branch, usually master.
+  final String baseRef = rootProject.property("gitBaseRef")
+  // -PgitNewRef sets the new git new reference to compare changes to. This is useful for testing the test selection method
+  // itself. Otherwise, comparing against current HEAD is what makes sense for CI.
+  final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD"
+
+  rootProject.ext {
+    it.changedFiles = getChangedFiles(baseRef, newRef)
+    useGitChanges = true
+  }
+
+  // The ignoredFiles FileTree selects any file that should not trigger any tasks.
+  final ignoredFiles = fileTree(rootProject.projectDir) {
+    include '.gitingore', '.editorconfig'
+    include '*.md', '**/*.md'
+    include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd'
+    include 'NOTICE'
+    include 'static-analysis.datadog.yml'
+  }
+  rootProject.changedFiles.each { File f ->
+    if (ignoredFiles.contains(f)) {
+      logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}")
+    }
+  }
+  rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) }
+
+  // The globalEffectsFile FileTree selects any file that should trigger all tasks, regardless of gradle dependency
+  // tracking.
+  final globalEffectFiles = fileTree(rootProject.projectDir) {
+    include '.circleci/**'
+    include 'build.gradle'
+    include 'gradle/**'
+  }
+
+  for (File f in rootProject.changedFiles) {
+    if (globalEffectFiles.contains(f)) {
+      logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)")
+      rootProject.useGitChanges = false
+      break
+    }
+  }
+
+  if (rootProject.useGitChanges) {
+    logger.warn("Git change tracking is enabled, base: ${baseRef}")
+
+    // Get all projects, sorted by descending path length.
+    final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() }
+    for (File f in rootProject.changedFiles) {
+      Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") }
+      if (p == null) {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no tasks will be skipped)")
+        rootProject.useGitChanges = false
+        break
+      }
+      final relPath = Paths.get(p.projectDir.path).relativize(f.toPath())
+      final pathComponents = relPath.collect({ it.toString() }).toList()
+      Map<Project, Set<String>> _affectedProjects = [:]
+      if (pathComponents.size() < 3) {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)")
+        _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add("all")
+      } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixturesClasses") {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)")
+        _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add("testFixturesClasses")
+      } else if (pathComponents[0] == "src" && pathComponents[1] == "testClasses") {
+        // TODO: We could include other variants here such as latestTest, etc.
+        logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)")
+        _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add("testClasses")
+      } else if (pathComponents[0] == "src" && pathComponents[1] == "jmhCompileGeneratedClasses") {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)")
+        _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add("jmhCompileGeneratedClasses")
+      } else {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)")
+        _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add("all")
+      }
+      rootProject.ext {
+        it.affectedProjects = _affectedProjects
+      }
+    }
+  }
+
+}
 
 def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {
-  def createRootTask = { rootTaskName, subProjTaskName ->
+  def createRootTask = { String rootTaskName, String subProjTaskName ->
     def coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
     tasks.register(rootTaskName) { aggTest ->
       subprojects { subproject ->
         if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
-          def testTask = subproject.tasks.findByName(subProjTaskName)
+          Task testTask = subproject.tasks.findByName(subProjTaskName)
+          boolean isAffected = true
           if (testTask != null) {
-            aggTest.dependsOn(testTask)
+            if (rootProject.useGitChanges) {
+              final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects"))
+              if (fileTrigger != null) {
+                logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})")
+              } else {
+                logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)")
+                isAffected = false
+              }
+            }
+            if (isAffected) {
+              aggTest.dependsOn(testTask)
+            }
           }
-          if (coverage) {
+          if (isAffected && coverage) {
             def coverageTask = subproject.tasks.findByName("jacocoTestReport")
             if (coverageTask != null) {
               aggTest.dependsOn(coverageTask)