Merge branch 'main' into dbartol/remove-actions-extractor

src/analyze.ts

@@ -26,7 +26,6 @@ import { getRepositoryNwoFromEnv } from "./repository";
 import { DatabaseCreationTimings, EventReport } from "./status-report";
 import { ToolsFeature } from "./tools-features";
 import { endTracingForCluster } from "./tracer-config";
-import { validateSarifFileSchema } from "./upload-lib";
 import * as util from "./util";
 import { BuildMode } from "./util";
 
@@ -42,6 +41,13 @@ export class CodeQLAnalysisError extends Error {
 }
 
 export interface QueriesStatusReport {
+  /**
+   * Time taken in ms to run queries for actions (or undefined if this language was not analyzed).
+   *
+   * The "builtin" designation is now outdated with the move to CLI config parsing: this is the time
+   * taken to run _all_ the queries.
+   */
+  analyze_builtin_queries_actions_duration_ms?: number;
   /**
    * Time taken in ms to run queries for cpp (or undefined if this language was not analyzed).
    *
@@ -98,6 +104,8 @@ export interface QueriesStatusReport {
    */
   analyze_builtin_queries_swift_duration_ms?: number;
 
+  /** Time taken in ms to interpret results for actions (or undefined if this language was not analyzed). */
+  interpret_results_actions_duration_ms?: number;
   /** Time taken in ms to interpret results for cpp (or undefined if this language was not analyzed). */
   interpret_results_cpp_duration_ms?: number;
   /** Time taken in ms to interpret results for csharp (or undefined if this language was not analyzed). */
@@ -498,7 +506,13 @@ function writeDiffRangeDataExtensionPack(
     actionsUtil.getTemporaryDirectory(),
     "pr-diff-range",
   );
-  fs.mkdirSync(diffRangeDir);
+
+  // We expect the Actions temporary directory to already exist, so are mainly
+  // using `recursive: true` to avoid errors if the directory already exists,
+  // for example if the analyze Action is run multiple times in the same job.
+  // This is not really something that is supported, but we make use of it in
+  // tests.
+  fs.mkdirSync(diffRangeDir, { recursive: true });
   fs.writeFileSync(
     path.join(diffRangeDir, "qlpack.yml"),
     `
@@ -517,6 +531,7 @@ extensions:
   - addsTo:
       pack: codeql/util
       extensible: restrictAlertsTo
+      checkPresence: false
     data:
 `;
 
@@ -614,7 +629,7 @@ export async function runQueries(
       logger.info(analysisSummary);
 
       if (await features.getValue(Feature.QaTelemetryEnabled)) {
-        const perQueryAlertCounts = getPerQueryAlertCounts(sarifFile, logger);
+        const perQueryAlertCounts = getPerQueryAlertCounts(sarifFile);
 
         const perQueryAlertCountEventReport: EventReport = {
           event: "codeql database interpret-results",
@@ -666,11 +681,7 @@ export async function runQueries(
   }
 
   /** Get an object with all queries and their counts parsed from a SARIF file path. */
-  function getPerQueryAlertCounts(
-    sarifPath: string,
-    log: Logger,
-  ): Record<string, number> {
-    validateSarifFileSchema(sarifPath, log);
+  function getPerQueryAlertCounts(sarifPath: string): Record<string, number> {
     const sarifObject = JSON.parse(
       fs.readFileSync(sarifPath, "utf8"),
     ) as util.SarifFile;

src/api-compatibility.json

@@ -1 +1 @@
-{"maximumVersion": "3.17", "minimumVersion": "3.12"}
+{"maximumVersion": "3.17", "minimumVersion": "3.13"}

src/codeql.ts

@@ -285,17 +285,17 @@ const CODEQL_MINIMUM_VERSION = "2.15.5";
 /**
  * This version will shortly become the oldest version of CodeQL that the Action will run with.
  */
-const CODEQL_NEXT_MINIMUM_VERSION = "2.15.5";
+const CODEQL_NEXT_MINIMUM_VERSION = "2.16.6";
 
 /**
  * This is the version of GHES that was most recently deprecated.
  */
-const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.11";
+const GHES_VERSION_MOST_RECENTLY_DEPRECATED = "3.12";
 
 /**
  * This is the deprecation date for the version of GHES that was most recently deprecated.
  */
-const GHES_MOST_RECENT_DEPRECATION_DATE = "2024-12-19";
+const GHES_MOST_RECENT_DEPRECATION_DATE = "2025-04-03";
 
 /** The CLI verbosity level to use for extraction in debug mode. */
 const EXTRACTION_DEBUG_MODE_VERBOSITY = "progress++";

src/debug-artifacts.ts

@@ -4,7 +4,7 @@ import * as path from "path";
 import * as artifact from "@actions/artifact";
 import * as artifactLegacy from "@actions/artifact-legacy";
 import * as core from "@actions/core";
-import AdmZip from "adm-zip";
+import archiver from "archiver";
 import del from "del";
 
 import { getOptionalInput, getTemporaryDirectory } from "./actions-util";
@@ -344,9 +344,24 @@ async function createPartialDatabaseBundle(
   if (fs.existsSync(databaseBundlePath)) {
     await del(databaseBundlePath, { force: true });
   }
-  const zip = new AdmZip();
-  zip.addLocalFolder(databasePath);
-  zip.writeZip(databaseBundlePath);
+  const output = fs.createWriteStream(databaseBundlePath);
+  const zip = archiver("zip");
+
+  zip.on("error", (err) => {
+    throw err;
+  });
+
+  zip.on("warning", (err) => {
+    // Ignore ENOENT warnings. There's nothing anyone can do about it.
+    if (err.code !== "ENOENT") {
+      throw err;
+    }
+  });
+
+  zip.pipe(output);
+  zip.directory(databasePath, false);
+  await zip.finalize();
+
   return databaseBundlePath;
 }
 

src/defaults.json

@@ -1,6 +1,6 @@
 {
-  "bundleVersion": "codeql-bundle-v2.20.7",
-  "cliVersion": "2.20.7",
-  "priorBundleVersion": "codeql-bundle-v2.20.6",
-  "priorCliVersion": "2.20.6"
+  "bundleVersion": "codeql-bundle-v2.21.3",
+  "cliVersion": "2.21.3",
+  "priorBundleVersion": "codeql-bundle-v2.21.2",
+  "priorCliVersion": "2.21.2"
 }

src/dependency-caching.ts

@@ -41,42 +41,44 @@ export function getJavaTempDependencyDir(): string {
 /**
  * Default caching configurations per language.
  */
-const CODEQL_DEFAULT_CACHE_CONFIG: { [language: string]: CacheConfig } = {
-  java: {
-    paths: [
-      // Maven
-      join(os.homedir(), ".m2", "repository"),
-      // Gradle
-      join(os.homedir(), ".gradle", "caches"),
-      // CodeQL Java build-mode: none
-      getJavaTempDependencyDir(),
-    ],
-    hash: [
-      // Maven
-      "**/pom.xml",
-      // Gradle
-      "**/*.gradle*",
-      "**/gradle-wrapper.properties",
-      "buildSrc/**/Versions.kt",
-      "buildSrc/**/Dependencies.kt",
-      "gradle/*.versions.toml",
-      "**/versions.properties",
-    ],
-  },
-  csharp: {
-    paths: [join(os.homedir(), ".nuget", "packages")],
-    hash: [
-      // NuGet
-      "**/packages.lock.json",
-      // Paket
-      "**/paket.lock",
-    ],
-  },
-  go: {
-    paths: [join(os.homedir(), "go", "pkg", "mod")],
-    hash: ["**/go.sum"],
-  },
-};
+function getDefaultCacheConfig(): { [language: string]: CacheConfig } {
+  return {
+    java: {
+      paths: [
+        // Maven
+        join(os.homedir(), ".m2", "repository"),
+        // Gradle
+        join(os.homedir(), ".gradle", "caches"),
+        // CodeQL Java build-mode: none
+        getJavaTempDependencyDir(),
+      ],
+      hash: [
+        // Maven
+        "**/pom.xml",
+        // Gradle
+        "**/*.gradle*",
+        "**/gradle-wrapper.properties",
+        "buildSrc/**/Versions.kt",
+        "buildSrc/**/Dependencies.kt",
+        "gradle/*.versions.toml",
+        "**/versions.properties",
+      ],
+    },
+    csharp: {
+      paths: [join(os.homedir(), ".nuget", "packages")],
+      hash: [
+        // NuGet
+        "**/packages.lock.json",
+        // Paket
+        "**/paket.lock",
+      ],
+    },
+    go: {
+      paths: [join(os.homedir(), "go", "pkg", "mod")],
+      hash: ["**/go.sum"],
+    },
+  };
+}
 
 async function makeGlobber(patterns: string[]): Promise<glob.Globber> {
   return glob.create(patterns.join("\n"));
@@ -96,7 +98,7 @@ export async function downloadDependencyCaches(
   const restoredCaches: Language[] = [];
 
   for (const language of languages) {
-    const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language];
+    const cacheConfig = getDefaultCacheConfig()[language];
 
     if (cacheConfig === undefined) {
       logger.info(
@@ -150,7 +152,7 @@ export async function downloadDependencyCaches(
  */
 export async function uploadDependencyCaches(config: Config, logger: Logger) {
   for (const language of config.languages) {
-    const cacheConfig = CODEQL_DEFAULT_CACHE_CONFIG[language];
+    const cacheConfig = getDefaultCacheConfig()[language];
 
     if (cacheConfig === undefined) {
       logger.info(

src/feature-flags.ts

@@ -112,8 +112,7 @@ export const featureConfig: Record<
   [Feature.DiffInformedQueries]: {
     defaultValue: false,
     envVar: "CODEQL_ACTION_DIFF_INFORMED_QUERIES",
-    minimumVersion: undefined,
-    toolsFeature: ToolsFeature.DatabaseInterpretResultsSupportsSarifRunProperty,
+    minimumVersion: "2.21.0",
   },
   [Feature.DisableCsharpBuildless]: {
     defaultValue: false,

src/init-action-post.ts

@@ -106,7 +106,9 @@ async function runWrapper() {
       ...uploadFailedSarifResult,
       job_status: initActionPostHelper.getFinalJobStatus(),
     };
+    logger.info("Sending status report for init-post step.");
     await sendStatusReport(statusReport);
+    logger.info("Status report sent for init-post step.");
   }
 }
 

src/init-action.ts

@@ -547,7 +547,8 @@ async function run() {
     );
     core.exportVariable(
       "CODEQL_THREADS",
-      getThreadsFlagValue(getOptionalInput("threads"), logger).toString(),
+      process.env["CODEQL_THREADS"] ||
+        getThreadsFlagValue(getOptionalInput("threads"), logger).toString(),
     );
 
     // Disable Kotlin extractor if feature flag set

src/start-proxy-action.ts

@@ -11,9 +11,9 @@ import { Credential, getCredentials } from "./start-proxy";
 import * as util from "./util";
 
 const UPDATEJOB_PROXY = "update-job-proxy";
-const UPDATEJOB_PROXY_VERSION = "v2.0.20241023203727";
+const UPDATEJOB_PROXY_VERSION = "v2.0.20250424171100";
 const UPDATEJOB_PROXY_URL_PREFIX =
-  "https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.18.1/";
+  "https://github.com/github/codeql-action/releases/download/codeql-bundle-v2.21.1/";
 const KEY_SIZE = 2048;
 const KEY_EXPIRY_YEARS = 2;
 

src/start-proxy.ts

@@ -18,10 +18,10 @@ const LANGUAGE_TO_REGISTRY_TYPE: Record<Language, string> = {
   python: "python_index",
   ruby: "rubygems_server",
   rust: "cargo_registry",
+  go: "goproxy_server",
   // We do not have an established proxy type for these languages, thus leaving empty.
   actions: "",
   cpp: "",
-  go: "",
   swift: "",
 } as const;
 

src/status-report.ts

@@ -29,6 +29,7 @@ import {
   assertNever,
   BuildMode,
   getErrorMessage,
+  getTestingEnvironment,
 } from "./util";
 
 export enum ActionName {
@@ -277,10 +278,10 @@ export async function createStatusReportBase(
     const runnerOs = getRequiredEnvParam("RUNNER_OS");
     const codeQlCliVersion = getCachedCodeQlVersion();
     const actionRef = process.env["GITHUB_ACTION_REF"] || "";
-    const testingEnvironment = process.env[EnvVar.TESTING_ENVIRONMENT] || "";
+    const testingEnvironment = getTestingEnvironment();
     // re-export the testing environment variable so that it is available to subsequent steps,
     // even if it was only set for this step
-    if (testingEnvironment !== "") {
+    if (testingEnvironment) {
       core.exportVariable(EnvVar.TESTING_ENVIRONMENT, testingEnvironment);
     }
     const isSteadyStateDefaultSetupRun =
@@ -303,7 +304,7 @@ export async function createStatusReportBase(
       started_at: workflowStartedAt,
       status,
       steady_state_default_setup: isSteadyStateDefaultSetupRun,
-      testing_environment: testingEnvironment,
+      testing_environment: testingEnvironment || "",
       workflow_name: workflowName,
       workflow_run_attempt: workflowRunAttempt,
       workflow_run_id: workflowRunID,

src/upload-lib.test.ts

@@ -17,14 +17,22 @@ test.beforeEach(() => {
 test("validateSarifFileSchema - valid", (t) => {
   const inputFile = `${__dirname}/../src/testdata/valid-sarif.sarif`;
   t.notThrows(() =>
-    uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)),
+    uploadLib.validateSarifFileSchema(
+      uploadLib.readSarifFile(inputFile),
+      inputFile,
+      getRunnerLogger(true),
+    ),
   );
 });
 
 test("validateSarifFileSchema - invalid", (t) => {
   const inputFile = `${__dirname}/../src/testdata/invalid-sarif.sarif`;
   t.throws(() =>
-    uploadLib.validateSarifFileSchema(inputFile, getRunnerLogger(true)),
+    uploadLib.validateSarifFileSchema(
+      uploadLib.readSarifFile(inputFile),
+      inputFile,
+      getRunnerLogger(true),
+    ),
   );
 });
 
@@ -314,7 +322,11 @@ test("accept results with invalid artifactLocation.uri value", (t) => {
   } as Logger;
 
   const sarifFile = `${__dirname}/../src/testdata/with-invalid-uri.sarif`;
-  uploadLib.validateSarifFileSchema(sarifFile, mockLogger);
+  uploadLib.validateSarifFileSchema(
+    uploadLib.readSarifFile(sarifFile),
+    sarifFile,
+    mockLogger,
+  );
 
   t.deepEqual(loggedMessages.length, 3);
   t.deepEqual(

src/upload-lib.ts

@@ -434,18 +434,35 @@ function countResultsInSarif(sarif: string): number {
   return numResults;
 }
 
-// Validates that the given file path refers to a valid SARIF file.
-// Throws an error if the file is invalid.
-export function validateSarifFileSchema(sarifFilePath: string, logger: Logger) {
-  logger.info(`Validating ${sarifFilePath}`);
-  let sarif;
+export function readSarifFile(sarifFilePath: string): SarifFile {
   try {
-    sarif = JSON.parse(fs.readFileSync(sarifFilePath, "utf8")) as SarifFile;
+    return JSON.parse(fs.readFileSync(sarifFilePath, "utf8")) as SarifFile;
   } catch (e) {
     throw new InvalidSarifUploadError(
       `Invalid SARIF. JSON syntax error: ${getErrorMessage(e)}`,
     );
   }
+}
+
+// Validates the given SARIF object and throws an error if the SARIF object is invalid.
+// The file path is only used in error messages to improve clarity.
+export function validateSarifFileSchema(
+  sarif: SarifFile,
+  sarifFilePath: string,
+  logger: Logger,
+) {
+  if (
+    areAllRunsProducedByCodeQL([sarif]) &&
+    // We want to validate CodeQL SARIF in testing environments.
+    !util.getTestingEnvironment()
+  ) {
+    logger.debug(
+      `Skipping SARIF schema validation for ${sarifFilePath} as all runs are produced by CodeQL.`,
+    );
+    return;
+  }
+
+  logger.info(`Validating ${sarifFilePath}`);
   // eslint-disable-next-line @typescript-eslint/no-require-imports
   const schema = require("../src/sarif-schema-2.1.0.json") as jsonschema.Schema;
 
@@ -551,41 +568,44 @@ export function buildPayload(
 }
 
 /**
- * Uploads a single SARIF file or a directory of SARIF files depending on what `sarifPath` refers
+ * Uploads a single SARIF file or a directory of SARIF files depending on what `inputSarifPath` refers
  * to.
  */
 export async function uploadFiles(
-  sarifPath: string,
+  inputSarifPath: string,
   checkoutPath: string,
   category: string | undefined,
   features: FeatureEnablement,
   logger: Logger,
 ): Promise<UploadResult> {
-  const sarifFiles = getSarifFilePaths(sarifPath);
+  const sarifPaths = getSarifFilePaths(inputSarifPath);
 
   logger.startGroup("Uploading results");
-  logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);
+  logger.info(`Processing sarif files: ${JSON.stringify(sarifPaths)}`);
 
   const gitHubVersion = await getGitHubVersion();
 
-  try {
+  let sarif: SarifFile;
+
+  if (sarifPaths.length > 1) {
     // Validate that the files we were asked to upload are all valid SARIF files
-    for (const file of sarifFiles) {
-      validateSarifFileSchema(file, logger);
+    for (const sarifPath of sarifPaths) {
+      const parsedSarif = readSarifFile(sarifPath);
+      validateSarifFileSchema(parsedSarif, sarifPath, logger);
     }
-  } catch (e) {
-    if (e instanceof SyntaxError) {
-      throw new InvalidSarifUploadError(e.message);
-    }
-    throw e;
+
+    sarif = await combineSarifFilesUsingCLI(
+      sarifPaths,
+      gitHubVersion,
+      features,
+      logger,
+    );
+  } else {
+    const sarifPath = sarifPaths[0];
+    sarif = readSarifFile(sarifPath);
+    validateSarifFileSchema(sarif, sarifPath, logger);
   }
 
-  let sarif = await combineSarifFilesUsingCLI(
-    sarifFiles,
-    gitHubVersion,
-    features,
-    logger,
-  );
   sarif = filterAlertsByDiffRange(logger, sarif);
   sarif = await fingerprints.addFingerprints(sarif, checkoutPath, logger);
 

src/util.ts

@@ -750,8 +750,8 @@ export function isGoodVersion(versionSpec: string) {
   return !BROKEN_VERSIONS.includes(versionSpec);
 }
 
-/*
- * Returns whether we are in test mode.
+/**
+ * Returns whether we are in test mode. This is used by CodeQL Action PR checks.
  *
  * In test mode, we don't upload SARIF results or status reports to the GitHub API.
  */
@@ -759,7 +759,20 @@ export function isInTestMode(): boolean {
   return process.env[EnvVar.TEST_MODE] === "true";
 }
 
-/*
+/**
+ * Get the testing environment.
+ *
+ * This is set if the CodeQL Action is running in a non-production environment.
+ */
+export function getTestingEnvironment(): string | undefined {
+  const testingEnvironment = process.env[EnvVar.TESTING_ENVIRONMENT] || "";
+  if (testingEnvironment === "") {
+    return undefined;
+  }
+  return testingEnvironment;
+}
+
+/**
  * Returns whether the path in the argument represents an existing directory.
  */
 export function doesDirectoryExist(dirPath: string): boolean {

src/workflow.ts

@@ -7,9 +7,12 @@ import * as yaml from "js-yaml";
 
 import * as api from "./api-client";
 import { CodeQL } from "./codeql";
-import { EnvVar } from "./environment";
 import { Logger } from "./logging";
-import { getRequiredEnvParam, isInTestMode } from "./util";
+import {
+  getRequiredEnvParam,
+  getTestingEnvironment,
+  isInTestMode,
+} from "./util";
 
 export interface WorkflowJobStep {
   name?: string;
@@ -358,10 +361,7 @@ function getInputOrThrow(
  * This allows us to test workflow parsing functionality as a CodeQL Action PR check.
  */
 function getAnalyzeActionName() {
-  if (
-    isInTestMode() ||
-    process.env[EnvVar.TESTING_ENVIRONMENT] === "codeql-action-pr-checks"
-  ) {
+  if (isInTestMode() || getTestingEnvironment() === "codeql-action-pr-checks") {
     return "./analyze";
   } else {
     return "github/codeql-action/analyze";