Merge branch 'main' into henrymercer/fix-ghae-setup-test

src/analyze.ts

@@ -262,6 +262,7 @@ export async function runQueries(
         logger.endGroup();
         logger.info(analysisSummary);
       } else {
+        // config was generated by the action, so must be interpreted by the action.
         logger.startGroup(`Running queries for ${language}`);
         const querySuitePaths: string[] = [];
         if (queries["builtin"].length > 0) {

src/codeql.test.ts

@@ -15,7 +15,7 @@ import { GitHubApiDetails } from "./api-client";
 import * as codeql from "./codeql";
 import { AugmentationProperties, Config } from "./config-utils";
 import * as defaults from "./defaults.json";
-import { Feature } from "./feature-flags";
+import { Feature, featureConfig } from "./feature-flags";
 import { Language } from "./languages";
 import { getRunnerLogger } from "./logging";
 import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
@@ -559,7 +559,7 @@ const injectedConfigMacro = test.macro({
       const codeqlObject = await codeql.getCodeQLForTesting();
       sinon
         .stub(codeqlObject, "getVersion")
-        .resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
+        .resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
 
       const thisStubConfig: Config = {
         ...stubConfig,
@@ -616,7 +616,7 @@ test(
   },
   {},
   {
-    packs: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
+    packs: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
   }
 );
 
@@ -637,7 +637,7 @@ test(
     packs: {
       javascript: [
         "codeql/something-else",
-        "codeql/javascript-experimental-atm-queries@~0.3.0",
+        "codeql/javascript-experimental-atm-queries@~0.4.0",
       ],
     },
   }
@@ -659,7 +659,7 @@ test(
   {
     packs: {
       cpp: ["codeql/something-else"],
-      javascript: ["codeql/javascript-experimental-atm-queries@~0.3.0"],
+      javascript: ["codeql/javascript-experimental-atm-queries@~0.4.0"],
     },
   }
 );
@@ -740,7 +740,7 @@ test(
     },
   },
   {
-    packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.3.0"],
+    packs: ["xxx", "yyy", "codeql/javascript-experimental-atm-queries@~0.4.0"],
   }
 );
 
@@ -872,7 +872,7 @@ test("does not use injected config", async (t: ExecutionContext<unknown>) => {
     const codeqlObject = await codeql.getCodeQLForTesting();
     sinon
       .stub(codeqlObject, "getVersion")
-      .resolves(codeql.CODEQL_VERSION_CONFIG_FILES);
+      .resolves(featureConfig[Feature.CliConfigFileEnabled].minimumVersion);
 
     await codeqlObject.databaseInitCluster(
       stubConfig,

src/codeql.ts

@@ -9,7 +9,7 @@ import * as yaml from "js-yaml";
 import * as semver from "semver";
 import { v4 as uuidV4 } from "uuid";
 
-import { isRunningLocalAction } from "./actions-util";
+import { getOptionalInput, isRunningLocalAction } from "./actions-util";
 import * as api from "./api-client";
 import { Config } from "./config-utils";
 import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
@@ -252,7 +252,6 @@ const CODEQL_MINIMUM_VERSION = "2.6.3";
  */
 const CODEQL_VERSION_CUSTOM_QUERY_HELP = "2.7.1";
 const CODEQL_VERSION_LUA_TRACER_CONFIG = "2.10.0";
-export const CODEQL_VERSION_CONFIG_FILES = "2.10.1";
 const CODEQL_VERSION_LUA_TRACING_GO_WINDOWS_FIXED = "2.10.4";
 export const CODEQL_VERSION_GHES_PACK_DOWNLOAD = "2.10.4";
 const CODEQL_VERSION_FILE_BASELINE_INFORMATION = "2.11.3";
@@ -885,24 +884,35 @@ async function getCodeQLForCmd(
         }
       }
 
+      // A config file is only generated if the CliConfigFileEnabled feature flag is enabled.
       const configLocation = await generateCodeScanningConfig(
         codeql,
         config,
         featureEnablement
       );
+      // Only pass external repository token if a config file is going to be parsed by the CLI.
+      let externalRepositoryToken: string | undefined;
       if (configLocation) {
         extraArgs.push(`--codescanning-config=${configLocation}`);
+        externalRepositoryToken = getOptionalInput("external-repository-token");
+        if (externalRepositoryToken) {
+          extraArgs.push("--external-repository-token-stdin");
+        }
       }
 
-      await runTool(cmd, [
-        "database",
-        "init",
-        "--db-cluster",
-        config.dbLocation,
-        `--source-root=${sourceRoot}`,
-        ...extraArgs,
-        ...getExtraOptionsFromEnv(["database", "init"]),
-      ]);
+      await runTool(
+        cmd,
+        [
+          "database",
+          "init",
+          "--db-cluster",
+          config.dbLocation,
+          `--source-root=${sourceRoot}`,
+          ...extraArgs,
+          ...getExtraOptionsFromEnv(["database", "init"]),
+        ],
+        { stdin: externalRepositoryToken }
+      );
     },
     async runAutobuild(language: Language) {
       const cmdName =
@@ -1335,7 +1345,11 @@ export function getExtraOptions(
  */
 const maxErrorSize = 20_000;
 
-async function runTool(cmd: string, args: string[] = []) {
+async function runTool(
+  cmd: string,
+  args: string[] = [],
+  opts: { stdin?: string } = {}
+) {
   let output = "";
   let error = "";
   const exitCode = await new toolrunner.ToolRunner(cmd, args, {
@@ -1354,6 +1368,7 @@ async function runTool(cmd: string, args: string[] = []) {
       },
     },
     ignoreReturnCode: true,
+    ...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
   }).exec();
   if (exitCode !== 0)
     throw new CommandInvocationError(cmd, args, exitCode, error, output);

src/config-utils.ts

@@ -582,16 +582,20 @@ async function parseQueryUses(
     );
   }
 
-  // Otherwise, must be a reference to another repo
-  await addRemoteQueries(
-    codeQL,
-    resultMap,
-    queryUses,
-    tempDir,
-    apiDetails,
-    logger,
-    configFile
-  );
+  // Otherwise, must be a reference to another repo.
+  // If config parsing is handled in CLI, then this repo will be downloaded
+  // later by the CLI.
+  if (!(await useCodeScanningConfigInCli(codeQL, featureEnablement))) {
+    await addRemoteQueries(
+      codeQL,
+      resultMap,
+      queryUses,
+      tempDir,
+      apiDetails,
+      logger,
+      configFile
+    );
+  }
   return false;
 }
 
@@ -1724,26 +1728,27 @@ export async function initConfig(
     );
   }
 
-  // The list of queries should not be empty for any language. If it is then
-  // it is a user configuration error.
-  for (const language of config.languages) {
-    const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
-    const hasCustomQueries = config.queries[language]?.custom.length > 0;
-    const hasPacks = (config.packs[language]?.length || 0) > 0;
-    if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
-      throw new Error(
-        `Did not detect any queries to run for ${language}. ` +
-          "Please make sure that the default queries are enabled, or you are specifying queries to run."
-      );
-    }
-  }
-
   // When using the codescanning config in the CLI, pack downloads
   // happen in the CLI during the `database init` command, so no need
   // to download them here.
   await logCodeScanningConfigInCli(codeQL, featureEnablement, logger);
 
   if (!(await useCodeScanningConfigInCli(codeQL, featureEnablement))) {
+    // The list of queries should not be empty for any language. If it is then
+    // it is a user configuration error.
+    // This check occurs in the CLI when it parses the config file.
+    for (const language of config.languages) {
+      const hasBuiltinQueries = config.queries[language]?.builtin.length > 0;
+      const hasCustomQueries = config.queries[language]?.custom.length > 0;
+      const hasPacks = (config.packs[language]?.length || 0) > 0;
+      if (!hasPacks && !hasBuiltinQueries && !hasCustomQueries) {
+        throw new Error(
+          `Did not detect any queries to run for ${language}. ` +
+            "Please make sure that the default queries are enabled, or you are specifying queries to run."
+        );
+      }
+    }
+
     const registries = parseRegistries(registriesInput);
     await downloadPacks(
       codeQL,

src/feature-flags.ts

@@ -43,7 +43,7 @@ export const featureConfig: Record<
   },
   [Feature.CliConfigFileEnabled]: {
     envVar: "CODEQL_PASS_CONFIG_TO_CLI",
-    minimumVersion: "2.11.1",
+    minimumVersion: "2.11.6",
   },
   [Feature.MlPoweredQueriesEnabled]: {
     envVar: "CODEQL_ML_POWERED_QUERIES",