Trace Go when Go extraction reconciliation is enabled

src/analyze-action.ts

@@ -19,7 +19,7 @@ import { runAutobuild } from "./autobuild";
 import { getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
-import { FeatureFlag, FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
+import { FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
 import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
@@ -146,12 +146,7 @@ async function runAutobuildIfLegacyGoWorkflow(
   if (!config.languages.includes(Language.go)) {
     return;
   }
-  if (
-    process.env["CODEQL_ACTION_RECONCILE_GO_EXTRACTION"] !== "true" &&
-    !(await featureFlags.getValue(
-      FeatureFlag.GolangExtractionReconciliationEnabled
-    ))
-  ) {
+  if (!(await util.isGoExtractionReconciliationEnabled(featureFlags))) {
     logger.debug(
       "Won't run Go autobuild since Go extraction reconciliation is not enabled."
     );

src/analyze.test.ts

@@ -494,6 +494,7 @@ for (const options of [
     const promise = createdDBForScannedLanguages(
       codeqlObject,
       stubConfig,
+      false, // Disable Go extraction reconciliation
       getRunnerLogger(true),
       createFeatureFlags(options.featureFlags)
     );

src/analyze.ts

@@ -121,6 +121,7 @@ async function setupPythonExtractor(logger: Logger) {
 export async function createdDBForScannedLanguages(
   codeql: CodeQL,
   config: configUtils.Config,
+  isGoExtractionReconciliationEnabled: boolean,
   logger: Logger,
   featureFlags: FeatureFlags
 ) {
@@ -130,7 +131,11 @@ export async function createdDBForScannedLanguages(
 
   for (const language of config.languages) {
     if (
-      isScannedLanguage(language, logger) &&
+      isScannedLanguage(
+        language,
+        isGoExtractionReconciliationEnabled,
+        logger
+      ) &&
       !dbIsFinalized(config, language, logger)
     ) {
       logger.startGroup(`Extracting ${language}`);
@@ -174,7 +179,13 @@ async function finalizeDatabaseCreation(
   const codeql = await getCodeQL(config.codeQLCmd);
 
   const extractionStart = performance.now();
-  await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
+  await createdDBForScannedLanguages(
+    codeql,
+    config,
+    await util.isGoExtractionReconciliationEnabled(featureFlags),
+    logger,
+    featureFlags
+  );
   const extractionTime = performance.now() - extractionStart;
 
   const trapImportStart = performance.now();
@@ -519,7 +530,11 @@ export async function runFinalize(
   // step.
   if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
     // Delete variables as specified by the end-tracing script
-    await endTracingForCluster(config, logger);
+    await endTracingForCluster(
+      config,
+      await util.isGoExtractionReconciliationEnabled(featureFlags),
+      logger
+    );
   } else {
     // Delete the tracer config env var to avoid tracing ourselves
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];

src/autobuild-action.ts

@@ -4,15 +4,25 @@ import {
   createStatusReportBase,
   getActionsStatus,
   getOptionalInput,
+  getRequiredInput,
   getTemporaryDirectory,
   sendStatusReport,
   StatusReportBase,
 } from "./actions-util";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { determineAutobuildLanguage, runAutobuild } from "./autobuild";
-import * as config_utils from "./config-utils";
+import * as configUtils from "./config-utils";
+import { GitHubFeatureFlags } from "./feature-flags";
 import { Language } from "./languages";
 import { getActionsLogger } from "./logging";
-import { checkActionVersion, initializeEnvironment, Mode } from "./util";
+import { parseRepositoryNwo } from "./repository";
+import {
+  checkActionVersion,
+  checkGitHubVersionInRange,
+  getRequiredEnvParam,
+  initializeEnvironment,
+  Mode,
+} from "./util";
 
 // eslint-disable-next-line import/no-commonjs
 const pkg = require("../package.json");
@@ -62,16 +72,35 @@ async function run() {
       return;
     }
 
-    const config = await config_utils.getConfig(
-      getTemporaryDirectory(),
+    const apiDetails = {
+      auth: getRequiredInput("token"),
+      externalRepoAuth: getOptionalInput("external-repository-token"),
+      url: getRequiredEnvParam("GITHUB_SERVER_URL"),
+      apiURL: getRequiredEnvParam("GITHUB_API_URL"),
+    };
+
+    const gitHubVersion = await getGitHubVersionActionsOnly();
+    checkGitHubVersionInRange(gitHubVersion, logger, Mode.actions);
+
+    const repositoryNwo = parseRepositoryNwo(
+      getRequiredEnvParam("GITHUB_REPOSITORY")
+    );
+
+    const featureFlags = new GitHubFeatureFlags(
+      gitHubVersion,
+      apiDetails,
+      repositoryNwo,
       logger
     );
+
+    const config = await configUtils.getConfig(getTemporaryDirectory(), logger);
     if (config === undefined) {
       throw new Error(
         "Config file could not be found at expected location. Has the 'init' action been called?"
       );
     }
-    language = determineAutobuildLanguage(config, logger);
+
+    language = await determineAutobuildLanguage(config, featureFlags, logger);
     if (language !== undefined) {
       const workingDirectory = getOptionalInput("working-directory");
       if (workingDirectory) {

src/autobuild.ts

@@ -1,18 +1,23 @@
 import { getCodeQL } from "./codeql";
-import * as config_utils from "./config-utils";
+import * as configUtils from "./config-utils";
+import { FeatureFlags } from "./feature-flags";
 import { Language, isTracedLanguage } from "./languages";
 import { Logger } from "./logging";
+import * as util from "./util";
 
-export function determineAutobuildLanguage(
-  config: config_utils.Config,
+export async function determineAutobuildLanguage(
+  config: configUtils.Config,
+  featureFlags: FeatureFlags,
   logger: Logger
-): Language | undefined {
+): Promise<Language | undefined> {
+  const isGoExtractionReconciliationEnabled =
+    await util.isGoExtractionReconciliationEnabled(featureFlags);
   // Attempt to find a language to autobuild
   // We want pick the dominant language in the repo from the ones we're able to build
   // The languages are sorted in order specified by user or by lines of code if we got
   // them from the GitHub API, so try to build the first language on the list.
   const autobuildLanguages = config.languages.filter((l) =>
-    isTracedLanguage(l, logger)
+    isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
   );
   const language = autobuildLanguages[0];
 
@@ -38,7 +43,7 @@ export function determineAutobuildLanguage(
 
 export async function runAutobuild(
   language: Language,
-  config: config_utils.Config,
+  config: configUtils.Config,
   logger: Logger
 ) {
   logger.startGroup(`Attempting to automatically build ${language} code`);

src/codeql.ts

@@ -777,8 +777,12 @@ async function getCodeQLForCmd(
       const extraArgs = config.languages.map(
         (language) => `--language=${language}`
       );
+      const isGoExtractionReconciliationEnabled =
+        await util.isGoExtractionReconciliationEnabled(featureFlags);
       if (
-        config.languages.filter((l) => isTracedLanguage(l, logger)).length > 0
+        config.languages.filter((l) =>
+          isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
+        ).length > 0
       ) {
         extraArgs.push("--begin-tracing");
         extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
@@ -800,7 +804,11 @@ async function getCodeQLForCmd(
             // Once we've released a fix, we should add a version gate based on the fixed version.
             !(
               config.languages.includes(Language.go) &&
-              isTracedLanguage(Language.go, logger) &&
+              isTracedLanguage(
+                Language.go,
+                isGoExtractionReconciliationEnabled,
+                logger
+              ) &&
               process.platform === "win32"
             )
           ) {

src/init.ts

@@ -134,7 +134,12 @@ export async function runInit(
       throw e;
     }
   }
-  return await getCombinedTracerConfig(config, codeql, logger);
+  return await getCombinedTracerConfig(
+    config,
+    codeql,
+    await util.isGoExtractionReconciliationEnabled(featureFlags),
+    logger
+  );
 }
 
 // Runs a powershell script to inject the tracer into a parent process

src/languages.test.ts

@@ -32,26 +32,40 @@ test("parseLanguage", async (t) => {
   t.deepEqual(parseLanguage(""), undefined);
 });
 
-test("isTracedLanguage", async (t) => {
-  const logger = getRunnerLogger(true);
+for (const isReconciliationOn of [false, true]) {
+  test(`isTracedLanguage (go reconciliation ${
+    isReconciliationOn ? "enabled" : "disabled"
+  }`, async (t) => {
+    const logger = getRunnerLogger(true);
 
-  t.true(isTracedLanguage(Language.cpp, logger));
-  t.true(isTracedLanguage(Language.java, logger));
-  t.true(isTracedLanguage(Language.csharp, logger));
+    t.true(isTracedLanguage(Language.cpp, isReconciliationOn, logger));
+    t.true(isTracedLanguage(Language.java, isReconciliationOn, logger));
+    t.true(isTracedLanguage(Language.csharp, isReconciliationOn, logger));
 
-  t.false(isTracedLanguage(Language.go, logger));
-  t.false(isTracedLanguage(Language.javascript, logger));
-  t.false(isTracedLanguage(Language.python, logger));
-});
+    t.is(
+      isTracedLanguage(Language.go, isReconciliationOn, logger),
+      isReconciliationOn
+    );
 
-test("isScannedLanguage", async (t) => {
-  const logger = getRunnerLogger(true);
+    t.false(isTracedLanguage(Language.javascript, isReconciliationOn, logger));
+    t.false(isTracedLanguage(Language.python, isReconciliationOn, logger));
+  });
 
-  t.false(isScannedLanguage(Language.cpp, logger));
-  t.false(isScannedLanguage(Language.java, logger));
-  t.false(isScannedLanguage(Language.csharp, logger));
+  test(`isScannedLanguage (go reconciliation ${
+    isReconciliationOn ? "enabled" : "disabled"
+  }`, async (t) => {
+    const logger = getRunnerLogger(true);
 
-  t.true(isScannedLanguage(Language.go, logger));
-  t.true(isScannedLanguage(Language.javascript, logger));
-  t.true(isScannedLanguage(Language.python, logger));
-});
+    t.false(isScannedLanguage(Language.cpp, isReconciliationOn, logger));
+    t.false(isScannedLanguage(Language.java, isReconciliationOn, logger));
+    t.false(isScannedLanguage(Language.csharp, isReconciliationOn, logger));
+
+    t.is(
+      isScannedLanguage(Language.go, isReconciliationOn, logger),
+      !isReconciliationOn
+    );
+
+    t.true(isScannedLanguage(Language.javascript, isReconciliationOn, logger));
+    t.true(isScannedLanguage(Language.python, isReconciliationOn, logger));
+  });
+}

src/languages.ts

@@ -40,7 +40,11 @@ export function parseLanguage(language: string): Language | undefined {
   return undefined;
 }
 
-export function isTracedLanguage(language: Language, logger: Logger): boolean {
+export function isTracedLanguage(
+  language: Language,
+  isGoExtractionReconciliationEnabled: boolean,
+  logger: Logger
+): boolean {
   if (process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "true") {
     logger.warning(
       "The CODEQL_EXTRACTOR_GO_BUILD_TRACING environment variable was set to 'true', but it must " +
@@ -50,13 +54,24 @@ export function isTracedLanguage(language: Language, logger: Logger): boolean {
     core.exportVariable("CODEQL_EXTRACTOR_GO_BUILD_TRACING", "on");
   }
 
+  const shouldTraceGo =
+    process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "on" ||
+    isGoExtractionReconciliationEnabled;
+
   return (
     ["cpp", "java", "csharp", "swift"].includes(language) ||
-    (process.env["CODEQL_EXTRACTOR_GO_BUILD_TRACING"] === "on" &&
-      language === Language.go)
+    (shouldTraceGo && language === Language.go)
   );
 }
 
-export function isScannedLanguage(language: Language, logger: Logger): boolean {
-  return !isTracedLanguage(language, logger);
+export function isScannedLanguage(
+  language: Language,
+  isGoExtractionReconciliationEnabled: boolean,
+  logger: Logger
+): boolean {
+  return !isTracedLanguage(
+    language,
+    isGoExtractionReconciliationEnabled,
+    logger
+  );
 }

src/runner.ts

@@ -373,7 +373,11 @@ program
           );
         }
       } else {
-        language = determineAutobuildLanguage(config, logger);
+        language = await determineAutobuildLanguage(
+          config,
+          createFeatureFlags([]),
+          logger
+        );
       }
       if (language !== undefined) {
         await runAutobuild(language, config, logger);

src/tracer-config.test.ts

@@ -331,7 +331,12 @@ test("getCombinedTracerConfig - return undefined when no languages are traced la
     });
 
     t.deepEqual(
-      await getCombinedTracerConfig(config, codeQL, getRunnerLogger(true)),
+      await getCombinedTracerConfig(
+        config,
+        codeQL,
+        false, // Disable Go extraction reconciliation
+        getRunnerLogger(true)
+      ),
       undefined
     );
   });
@@ -366,6 +371,7 @@ test("getCombinedTracerConfig - valid spec file", async (t) => {
     const result = await getCombinedTracerConfig(
       config,
       codeQL,
+      false, // Disable Go extraction reconciliation
       getRunnerLogger(true)
     );
     t.notDeepEqual(result, undefined);

src/tracer-config.ts

@@ -23,10 +23,16 @@ const CRITICAL_TRACER_VARS = new Set([
 
 export async function endTracingForCluster(
   config: configUtils.Config,
+  isGoExtractionReconciliationEnabled: boolean,
   logger: Logger
 ): Promise<void> {
   // If there are no traced languages, we don't need to do anything.
-  if (!config.languages.some((l) => isTracedLanguage(l, logger))) return;
+  if (
+    !config.languages.some((l) =>
+      isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
+    )
+  )
+    return;
 
   const envVariablesFile = path.resolve(
     config.dbLocation,
@@ -226,11 +232,12 @@ export function concatTracerConfigs(
 export async function getCombinedTracerConfig(
   config: configUtils.Config,
   codeql: CodeQL,
+  isGoExtractionReconciliationEnabled: boolean,
   logger: Logger
 ): Promise<TracerConfig | undefined> {
   // Abort if there are no traced languages as there's nothing to do
   const tracedLanguages = config.languages.filter((l) =>
-    isTracedLanguage(l, logger)
+    isTracedLanguage(l, isGoExtractionReconciliationEnabled, logger)
   );
   if (tracedLanguages.length === 0) {
     return undefined;

src/util.ts

@@ -20,6 +20,7 @@ import {
   parsePacksSpecification,
   prettyPrintPack,
 } from "./config-utils";
+import { FeatureFlag, FeatureFlags } from "./feature-flags";
 import { Language } from "./languages";
 import { Logger } from "./logging";
 
@@ -817,3 +818,14 @@ export function listFolder(dir: string): string[] {
   }
   return files;
 }
+
+export async function isGoExtractionReconciliationEnabled(
+  featureFlags: FeatureFlags
+): Promise<boolean> {
+  return (
+    process.env["CODEQL_ACTION_RECONCILE_GO_EXTRACTION"] === "true" ||
+    (await featureFlags.getValue(
+      FeatureFlag.GolangExtractionReconciliationEnabled
+    ))
+  );
+}