Honor the Lua tracer FF for database trace-command invocations for scanned languages.

In theory, a scanned language will not setup the build tracer, and so
shouldn't care about lua versus legacy tracing. However, `go` is a
special case where the autobuilder runs under the build tracer, that
then gets disabled immediately again, unless a special environment
variable is used.
Therefore, we need to thread through the feature flag to this
`database trace-command` invocation. For other scanned languages,
this should be a no-op, as no tracing is ever set up.

lib/analyze-action.js

@@ -26,9 +26,11 @@ const artifact = __importStar(require("@actions/artifact"));
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
+const api_client_1 = require("./api-client");
 const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const database_upload_1 = require("./database-upload");
+const feature_flags_1 = require("./feature-flags");
 const logging_1 = require("./logging");
 const repository_1 = require("./repository");
 const upload_lib = __importStar(require("./upload-lib"));
@@ -76,7 +78,9 @@ async function run() {
         const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
         const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
+        const featureFlags = new feature_flags_1.GitHubFeatureFlags(gitHubVersion, apiDetails, repositoryNwo, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, featureFlags);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
             if (config.debugMode) {

lib/analyze.js

@@ -68,7 +68,7 @@ async function setupPythonExtractor(logger) {
     logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
     process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
-async function createdDBForScannedLanguages(config, logger) {
+async function createdDBForScannedLanguages(config, logger, featureFlags) {
     // Insert the LGTM_INDEX_X env vars at this point so they are set when
     // we extract any scanned languages.
     analysisPaths.includeAndExcludeAnalysisPaths(config);
@@ -80,7 +80,7 @@ async function createdDBForScannedLanguages(config, logger) {
             if (language === languages_1.Language.python) {
                 await setupPythonExtractor(logger);
             }
-            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language);
+            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language, featureFlags);
             logger.endGroup();
         }
     }
@@ -96,8 +96,8 @@ function dbIsFinalized(config, language, logger) {
         return false;
     }
 }
-async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger) {
-    await createdDBForScannedLanguages(config, logger);
+async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags) {
+    await createdDBForScannedLanguages(config, logger, featureFlags);
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     for (const language of config.languages) {
         if (dbIsFinalized(config, language, logger)) {
@@ -238,7 +238,7 @@ exports.runQueries = runQueries;
 function createQuerySuiteContents(queries) {
     return queries.map((q) => `- query: ${q}`).join("\n");
 }
-async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
+async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, featureFlags) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     if (await util.codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
         // Delete variables as specified by the end-tracing script
@@ -257,7 +257,7 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
         }
     }
     await fs.promises.mkdir(outputDir, { recursive: true });
-    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags);
 }
 exports.runFinalize = runFinalize;
 async function runCleanup(config, cleanupLevel, logger) {

lib/codeql.js

@@ -510,7 +510,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             ].join(" ");
             await runTool(autobuildCmd);
         },
-        async extractScannedLanguage(databasePath, language) {
+        async extractScannedLanguage(databasePath, language, featureFlags) {
             // Get extractor location
             let extractorPath = "";
             await new toolrunner.ToolRunner(cmd, [
@@ -533,10 +533,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             // Set trace command
             const ext = process.platform === "win32" ? ".cmd" : ".sh";
             const traceCommand = path.resolve(JSON.parse(extractorPath), "tools", `autobuild${ext}`);
+            const extraArgs = [];
+            if (await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)) {
+                if (await featureFlags.getValue(feature_flags_1.FeatureFlag.LuaTracerConfigEnabled)) {
+                    extraArgs.push("--internal-use-lua-tracing");
+                }
+                else {
+                    extraArgs.push("--no-internal-use-lua-tracing");
+                }
+            }
             // Run trace command
             await (0, toolrunner_error_catcher_1.toolrunnerErrorCatcher)(cmd, [
                 "database",
                 "trace-command",
+                ...extraArgs,
                 ...getExtraOptionsFromEnv(["database", "trace-command"]),
                 databasePath,
                 "--",

lib/runner.js

@@ -297,7 +297,7 @@ program
         }
         const threads = (0, util_1.getThreadsFlag)(cmd.threads || initEnv["CODEQL_THREADS"], logger);
         const memory = (0, util_1.getMemoryFlag)(cmd.ram || initEnv["CODEQL_RAM"]);
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, (0, feature_flags_1.createFeatureFlags)([]));
         await (0, analyze_1.runQueries)(outputDir, memory, (0, util_1.getAddSnippetsFlag)(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {
             logger.info("Not uploading results");

src/analyze-action.ts

@@ -12,9 +12,11 @@ import {
   runQueries,
   runFinalize,
 } from "./analyze";
+import { getGitHubVersionActionsOnly } from "./api-client";
 import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
+import { GitHubFeatureFlags } from "./feature-flags";
 import { getActionsLogger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
@@ -112,7 +114,16 @@ async function run() {
       util.getRequiredEnvParam("GITHUB_REPOSITORY")
     );
 
-    await runFinalize(outputDir, threads, memory, config, logger);
+    const gitHubVersion = await getGitHubVersionActionsOnly();
+
+    const featureFlags = new GitHubFeatureFlags(
+      gitHubVersion,
+      apiDetails,
+      repositoryNwo,
+      logger
+    );
+
+    await runFinalize(outputDir, threads, memory, config, logger, featureFlags);
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
         outputDir,

src/analyze.ts

@@ -13,6 +13,7 @@ import {
 } from "./codeql";
 import * as configUtils from "./config-utils";
 import { countLoc } from "./count-loc";
+import { FeatureFlags } from "./feature-flags";
 import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import * as sharedEnv from "./shared-environment";
@@ -116,7 +117,8 @@ async function setupPythonExtractor(logger: Logger) {
 
 async function createdDBForScannedLanguages(
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   // Insert the LGTM_INDEX_X env vars at this point so they are set when
   // we extract any scanned languages.
@@ -136,7 +138,8 @@ async function createdDBForScannedLanguages(
 
       await codeql.extractScannedLanguage(
         util.getCodeQLDatabasePath(config, language),
-        language
+        language,
+        featureFlags
       );
       logger.endGroup();
     }
@@ -166,9 +169,10 @@ async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
   memoryFlag: string,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
-  await createdDBForScannedLanguages(config, logger);
+  await createdDBForScannedLanguages(config, logger, featureFlags);
 
   const codeql = await getCodeQL(config.codeQLCmd);
   for (const language of config.languages) {
@@ -425,7 +429,8 @@ export async function runFinalize(
   threadsFlag: string,
   memoryFlag: string,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  featureFlags: FeatureFlags
 ) {
   const codeql = await getCodeQL(config.codeQLCmd);
   if (await util.codeQlVersionAbove(codeql, CODEQL_VERSION_NEW_TRACING)) {
@@ -445,7 +450,13 @@ export async function runFinalize(
   }
   await fs.promises.mkdir(outputDir, { recursive: true });
 
-  await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+  await finalizeDatabaseCreation(
+    config,
+    threadsFlag,
+    memoryFlag,
+    logger,
+    featureFlags
+  );
 }
 
 export async function runCleanup(

src/codeql.ts

@@ -95,7 +95,11 @@ export interface CodeQL {
    * Extract code for a scanned language using 'codeql database trace-command'
    * and running the language extractor.
    */
-  extractScannedLanguage(database: string, language: Language): Promise<void>;
+  extractScannedLanguage(
+    database: string,
+    language: Language,
+    featureFlags: FeatureFlags
+  ): Promise<void>;
   /**
    * Finalize a database using 'codeql database finalize'.
    */
@@ -789,7 +793,11 @@ async function getCodeQLForCmd(
 
       await runTool(autobuildCmd);
     },
-    async extractScannedLanguage(databasePath: string, language: Language) {
+    async extractScannedLanguage(
+      databasePath: string,
+      language: Language,
+      featureFlags: FeatureFlags
+    ) {
       // Get extractor location
       let extractorPath = "";
       await new toolrunner.ToolRunner(
@@ -821,6 +829,16 @@ async function getCodeQLForCmd(
         "tools",
         `autobuild${ext}`
       );
+      const extraArgs: string[] = [];
+      if (
+        await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)
+      ) {
+        if (await featureFlags.getValue(FeatureFlag.LuaTracerConfigEnabled)) {
+          extraArgs.push("--internal-use-lua-tracing");
+        } else {
+          extraArgs.push("--no-internal-use-lua-tracing");
+        }
+      }
 
       // Run trace command
       await toolrunnerErrorCatcher(
@@ -828,6 +846,7 @@ async function getCodeQLForCmd(
         [
           "database",
           "trace-command",
+          ...extraArgs,
           ...getExtraOptionsFromEnv(["database", "trace-command"]),
           databasePath,
           "--",

src/runner.ts

@@ -501,7 +501,14 @@ program
         logger
       );
       const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
-      await runFinalize(outputDir, threads, memory, config, logger);
+      await runFinalize(
+        outputDir,
+        threads,
+        memory,
+        config,
+        logger,
+        createFeatureFlags([])
+      );
       await runQueries(
         outputDir,
         memory,