Merge branch 'main' into update-bundle/codeql-bundle-v2.20.5

.pre-commit-config.yaml

@@ -16,5 +16,5 @@ repos:
         name: Synchronize PR check workflows
         files: ^.github/workflows/__.*\.yml$|^pr-checks
         language: system
-        entry: python3 pr-checks/sync.py
+        entry: pr-checks/sync.sh
         pass_filenames: false

CHANGELOG.md

@@ -4,8 +4,8 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 
 ## [UNRELEASED]
 
-- Update the action to prefer `gtar` over `tar` to make zstd archive extraction more robust. [2767](https://github.com/github/codeql-action/pull/2767)
 - Update default CodeQL bundle version to 2.20.5. [#2772](https://github.com/github/codeql-action/pull/2772)
+- Address an issue where the CodeQL Bundle would occasionally fail to decompress on macOS. [#2768](https://github.com/github/codeql-action/pull/2768)
 
 ## 3.28.9 - 07 Feb 2025
 

justfile

@@ -0,0 +1,10 @@
+# Sync generated files (javascript and PR checks)
+sync: build update-pr-checks
+
+# Perform all necessary steps to update the PR checks
+update-pr-checks:
+    pr-checks/sync.sh
+
+# Transpile typescript code into javascript
+build:
+    npm run build

lib/analyze.js

@@ -55,6 +55,7 @@ const api_client_1 = require("./api-client");
 const autobuild_1 = require("./autobuild");
 const codeql_1 = require("./codeql");
 const diagnostics_1 = require("./diagnostics");
+const diff_filtering_utils_1 = require("./diff-filtering-utils");
 const environment_1 = require("./environment");
 const feature_flags_1 = require("./feature-flags");
 const languages_1 = require("./languages");
@@ -368,6 +369,9 @@ extensions:
     const extensionFilePath = path.join(diffRangeDir, "pr-diff-range.yml");
     fs.writeFileSync(extensionFilePath, extensionContents);
     logger.debug(`Wrote pr-diff-range extension pack to ${extensionFilePath}:\n${extensionContents}`);
+    // Write the diff ranges to a JSON file, for action-side alert filtering by the
+    // upload-lib module.
+    (0, diff_filtering_utils_1.writeDiffRangesJsonFile)(logger, ranges);
     return diffRangeDir;
 }
 // Runs queries and creates sarif files in the given folder

lib/diff-filtering-utils.js

@@ -0,0 +1,60 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeDiffRangesJsonFile = writeDiffRangesJsonFile;
+exports.readDiffRangesJsonFile = readDiffRangesJsonFile;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const actionsUtil = __importStar(require("./actions-util"));
+function getDiffRangesJsonFilePath() {
+    return path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range.json");
+}
+function writeDiffRangesJsonFile(logger, ranges) {
+    const jsonContents = JSON.stringify(ranges, null, 2);
+    const jsonFilePath = getDiffRangesJsonFilePath();
+    fs.writeFileSync(jsonFilePath, jsonContents);
+    logger.debug(`Wrote pr-diff-range JSON file to ${jsonFilePath}:\n${jsonContents}`);
+}
+function readDiffRangesJsonFile(logger) {
+    const jsonFilePath = getDiffRangesJsonFilePath();
+    if (!fs.existsSync(jsonFilePath)) {
+        logger.debug(`Diff ranges JSON file does not exist at ${jsonFilePath}`);
+        return undefined;
+    }
+    const jsonContents = fs.readFileSync(jsonFilePath, "utf8");
+    logger.debug(`Read pr-diff-range JSON file from ${jsonFilePath}:\n${jsonContents}`);
+    return JSON.parse(jsonContents);
+}
+//# sourceMappingURL=diff-filtering-utils.js.map

lib/diff-filtering-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"diff-filtering-utils.js","sourceRoot":"","sources":["../src/diff-filtering-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgBA,0DAUC;AAED,wDAaC;AAzCD,uCAAyB;AACzB,2CAA6B;AAE7B,4DAA8C;AAS9C,SAAS,yBAAyB;IAChC,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,qBAAqB,EAAE,EAAE,oBAAoB,CAAC,CAAC;AAC9E,CAAC;AAED,SAAgB,uBAAuB,CACrC,MAAc,EACd,MAAwB;IAExB,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,yBAAyB,EAAE,CAAC;IACjD,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,YAAY,CAAC,CAAC;IAC7C,MAAM,CAAC,KAAK,CACV,oCAAoC,YAAY,MAAM,YAAY,EAAE,CACrE,CAAC;AACJ,CAAC;AAED,SAAgB,sBAAsB,CACpC,MAAc;IAEd,MAAM,YAAY,GAAG,yBAAyB,EAAE,CAAC;IACjD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QACjC,MAAM,CAAC,KAAK,CAAC,2CAA2C,YAAY,EAAE,CAAC,CAAC;QACxE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC3D,MAAM,CAAC,KAAK,CACV,qCAAqC,YAAY,MAAM,YAAY,EAAE,CACtE,CAAC;IACF,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAqB,CAAC;AACtD,CAAC"}

lib/tar.js

@@ -48,8 +48,8 @@ const actions_util_1 = require("./actions-util");
 const util_1 = require("./util");
 const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
 const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
-async function getTarVersion(programName) {
-    const tar = await io.which(programName, true);
+async function getTarVersion() {
+    const tar = await io.which("tar", true);
     let stdout = "";
     const exitCode = await new toolrunner_1.ToolRunner(tar, ["--version"], {
         listeners: {
@@ -59,46 +59,31 @@ async function getTarVersion(programName) {
         },
     }).exec();
     if (exitCode !== 0) {
-        throw new Error(`Failed to call ${programName} --version`);
+        throw new Error("Failed to call tar --version");
     }
     // Return whether this is GNU tar or BSD tar, and the version number
     if (stdout.includes("GNU tar")) {
         const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
         if (!match || !match[1]) {
-            throw new Error(`Failed to parse output of ${programName} --version.`);
+            throw new Error("Failed to parse output of tar --version.");
         }
-        return { name: programName, type: "gnu", version: match[1] };
+        return { type: "gnu", version: match[1] };
     }
     else if (stdout.includes("bsdtar")) {
         const match = stdout.match(/bsdtar ([0-9.]+)/);
         if (!match || !match[1]) {
-            throw new Error(`Failed to parse output of ${programName} --version.`);
+            throw new Error("Failed to parse output of tar --version.");
         }
-        return { name: programName, type: "bsd", version: match[1] };
+        return { type: "bsd", version: match[1] };
     }
     else {
         throw new Error("Unknown tar version");
     }
 }
-async function pickTarCommand() {
-    // bsdtar 3.5.3 on the macos-14 (arm) action runner image is prone to crash with the following
-    // error messages when extracting zstd archives:
-    //
-    //   tar: Child process exited with status 1
-    //   tar: Error exit delayed from previous errors.
-    //
-    // To avoid this problem, prefer GNU tar under the name "gtar" if it is available.
-    try {
-        return await getTarVersion("gtar");
-    }
-    catch {
-        return await getTarVersion("tar");
-    }
-}
 async function isZstdAvailable(logger) {
     const foundZstdBinary = await (0, util_1.isBinaryAccessible)("zstd", logger);
     try {
-        const tarVersion = await pickTarCommand();
+        const tarVersion = await getTarVersion();
         const { type, version } = tarVersion;
         logger.info(`Found ${type} tar version ${version}.`);
         switch (type) {
@@ -174,9 +159,9 @@ async function extractTarZst(tar, dest, tarVersion, logger) {
             args.push("--overwrite");
         }
         args.push("-f", tar instanceof stream.Readable ? "-" : tar, "-C", dest);
-        process.stdout.write(`[command]${tarVersion.name} ${args.join(" ")}\n`);
+        process.stdout.write(`[command]tar ${args.join(" ")}\n`);
         await new Promise((resolve, reject) => {
-            const tarProcess = (0, child_process_1.spawn)(tarVersion.name, args, { stdio: "pipe" });
+            const tarProcess = (0, child_process_1.spawn)("tar", args, { stdio: "pipe" });
             let stdout = "";
             tarProcess.stdout?.on("data", (data) => {
                 stdout += data.toString();
@@ -198,7 +183,7 @@ async function extractTarZst(tar, dest, tarVersion, logger) {
             }
             tarProcess.on("exit", (code) => {
                 if (code !== 0) {
-                    reject(new actions_util_1.CommandInvocationError(tarVersion.name, args, code ?? undefined, stdout, stderr));
+                    reject(new actions_util_1.CommandInvocationError("tar", args, code ?? undefined, stdout, stderr));
                 }
                 resolve();
             });

lib/upload-lib.js

@@ -58,6 +58,7 @@ const api = __importStar(require("./api-client"));
 const api_client_1 = require("./api-client");
 const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
+const diff_filtering_utils_1 = require("./diff-filtering-utils");
 const environment_1 = require("./environment");
 const fingerprints = __importStar(require("./fingerprints"));
 const gitUtils = __importStar(require("./git-utils"));
@@ -412,6 +413,7 @@ async function uploadFiles(sarifPath, checkoutPath, category, features, logger)
         validateSarifFileSchema(file, logger);
     }
     let sarif = await combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, logger);
+    sarif = filterAlertsByDiffRange(logger, sarif);
     sarif = await fingerprints.addFingerprints(sarif, checkoutPath, logger);
     const analysisKey = await api.getAnalysisKey();
     const environment = actionsUtil.getRequiredInput("matrix");
@@ -607,4 +609,43 @@ class InvalidSarifUploadError extends Error {
     }
 }
 exports.InvalidSarifUploadError = InvalidSarifUploadError;
+function filterAlertsByDiffRange(logger, sarif) {
+    const diffRanges = (0, diff_filtering_utils_1.readDiffRangesJsonFile)(logger);
+    if (!diffRanges?.length) {
+        return sarif;
+    }
+    const checkoutPath = actionsUtil.getRequiredInput("checkout_path");
+    for (const run of sarif.runs) {
+        if (run.results) {
+            run.results = run.results.filter((result) => {
+                const locations = [
+                    ...(result.locations || []).map((loc) => loc.physicalLocation),
+                    ...(result.relatedLocations || []).map((loc) => loc.physicalLocation),
+                ];
+                return locations.some((physicalLocation) => {
+                    const locationUri = physicalLocation?.artifactLocation?.uri;
+                    const locationStartLine = physicalLocation?.region?.startLine;
+                    if (!locationUri || locationStartLine === undefined) {
+                        return false;
+                    }
+                    // CodeQL always uses forward slashes as the path separator, so on Windows we
+                    // need to replace any backslashes with forward slashes.
+                    const locationPath = path
+                        .join(checkoutPath, locationUri)
+                        .replaceAll(path.sep, "/");
+                    // Alert filtering here replicates the same behavior as the restrictAlertsTo
+                    // extensible predicate in CodeQL. See the restrictAlertsTo documentation
+                    // https://codeql.github.com/codeql-standard-libraries/csharp/codeql/util/AlertFiltering.qll/predicate.AlertFiltering$restrictAlertsTo.3.html
+                    // for more details, such as why the filtering applies only to the first line
+                    // of an alert location.
+                    return diffRanges.some((range) => range.path === locationPath &&
+                        ((range.startLine <= locationStartLine &&
+                            range.endLine >= locationStartLine) ||
+                            (range.startLine === 0 && range.endLine === 0)));
+                });
+            });
+        }
+    }
+    return sarif;
+}
 //# sourceMappingURL=upload-lib.js.map

pr-checks/justfile

@@ -1,6 +1 @@
-# Perform all necessary steps to update the PR checks
-update-pr-checks:
-    python3 -m venv env
-    source env/bin/activate
-    pip3 install ruamel.yaml
-    python3 sync.py
+set fallback := true

pr-checks/sync.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$0")"
+python3 -m venv env
+source env/bin/activate
+pip3 install ruamel.yaml
+python3 sync.py
+

src/analyze.ts

@@ -12,6 +12,10 @@ import { setupCppAutobuild } from "./autobuild";
 import { CodeQL, getCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { addDiagnostic, makeDiagnostic } from "./diagnostics";
+import {
+  DiffThunkRange,
+  writeDiffRangesJsonFile,
+} from "./diff-filtering-utils";
 import { EnvVar } from "./environment";
 import { FeatureEnablement, Feature } from "./feature-flags";
 import { isScannedLanguage, Language } from "./languages";
@@ -284,12 +288,6 @@ export async function setupDiffInformedQueryRun(
   );
 }
 
-interface DiffThunkRange {
-  path: string;
-  startLine: number;
-  endLine: number;
-}
-
 /**
  * Return the file line ranges that were added or modified in the pull request.
  *
@@ -537,6 +535,10 @@ extensions:
     `Wrote pr-diff-range extension pack to ${extensionFilePath}:\n${extensionContents}`,
   );
 
+  // Write the diff ranges to a JSON file, for action-side alert filtering by the
+  // upload-lib module.
+  writeDiffRangesJsonFile(logger, ranges);
+
   return diffRangeDir;
 }
 

src/diff-filtering-utils.ts

@@ -0,0 +1,42 @@
+import * as fs from "fs";
+import * as path from "path";
+
+import * as actionsUtil from "./actions-util";
+import { Logger } from "./logging";
+
+export interface DiffThunkRange {
+  path: string;
+  startLine: number;
+  endLine: number;
+}
+
+function getDiffRangesJsonFilePath(): string {
+  return path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range.json");
+}
+
+export function writeDiffRangesJsonFile(
+  logger: Logger,
+  ranges: DiffThunkRange[],
+): void {
+  const jsonContents = JSON.stringify(ranges, null, 2);
+  const jsonFilePath = getDiffRangesJsonFilePath();
+  fs.writeFileSync(jsonFilePath, jsonContents);
+  logger.debug(
+    `Wrote pr-diff-range JSON file to ${jsonFilePath}:\n${jsonContents}`,
+  );
+}
+
+export function readDiffRangesJsonFile(
+  logger: Logger,
+): DiffThunkRange[] | undefined {
+  const jsonFilePath = getDiffRangesJsonFilePath();
+  if (!fs.existsSync(jsonFilePath)) {
+    logger.debug(`Diff ranges JSON file does not exist at ${jsonFilePath}`);
+    return undefined;
+  }
+  const jsonContents = fs.readFileSync(jsonFilePath, "utf8");
+  logger.debug(
+    `Read pr-diff-range JSON file from ${jsonFilePath}:\n${jsonContents}`,
+  );
+  return JSON.parse(jsonContents) as DiffThunkRange[];
+}

src/tar.ts

@@ -15,13 +15,12 @@ const MIN_REQUIRED_BSD_TAR_VERSION = "3.4.3";
 const MIN_REQUIRED_GNU_TAR_VERSION = "1.31";
 
 export type TarVersion = {
-  name: string;
   type: "gnu" | "bsd";
   version: string;
 };
 
-async function getTarVersion(programName: string): Promise<TarVersion> {
-  const tar = await io.which(programName, true);
+async function getTarVersion(): Promise<TarVersion> {
+  const tar = await io.which("tar", true);
   let stdout = "";
   const exitCode = await new ToolRunner(tar, ["--version"], {
     listeners: {
@@ -31,43 +30,28 @@ async function getTarVersion(programName: string): Promise<TarVersion> {
     },
   }).exec();
   if (exitCode !== 0) {
-    throw new Error(`Failed to call ${programName} --version`);
+    throw new Error("Failed to call tar --version");
   }
   // Return whether this is GNU tar or BSD tar, and the version number
   if (stdout.includes("GNU tar")) {
     const match = stdout.match(/tar \(GNU tar\) ([0-9.]+)/);
     if (!match || !match[1]) {
-      throw new Error(`Failed to parse output of ${programName} --version.`);
+      throw new Error("Failed to parse output of tar --version.");
     }
 
-    return { name: programName, type: "gnu", version: match[1] };
+    return { type: "gnu", version: match[1] };
   } else if (stdout.includes("bsdtar")) {
     const match = stdout.match(/bsdtar ([0-9.]+)/);
     if (!match || !match[1]) {
-      throw new Error(`Failed to parse output of ${programName} --version.`);
+      throw new Error("Failed to parse output of tar --version.");
     }
 
-    return { name: programName, type: "bsd", version: match[1] };
+    return { type: "bsd", version: match[1] };
   } else {
     throw new Error("Unknown tar version");
   }
 }
 
-async function pickTarCommand(): Promise<TarVersion> {
-  // bsdtar 3.5.3 on the macos-14 (arm) action runner image is prone to crash with the following
-  // error messages when extracting zstd archives:
-  //
-  //   tar: Child process exited with status 1
-  //   tar: Error exit delayed from previous errors.
-  //
-  // To avoid this problem, prefer GNU tar under the name "gtar" if it is available.
-  try {
-    return await getTarVersion("gtar");
-  } catch {
-    return await getTarVersion("tar");
-  }
-}
-
 export interface ZstdAvailability {
   available: boolean;
   foundZstdBinary: boolean;
@@ -79,7 +63,7 @@ export async function isZstdAvailable(
 ): Promise<ZstdAvailability> {
   const foundZstdBinary = await isBinaryAccessible("zstd", logger);
   try {
-    const tarVersion = await pickTarCommand();
+    const tarVersion = await getTarVersion();
     const { type, version } = tarVersion;
     logger.info(`Found ${type} tar version ${version}.`);
     switch (type) {
@@ -187,10 +171,10 @@ export async function extractTarZst(
 
     args.push("-f", tar instanceof stream.Readable ? "-" : tar, "-C", dest);
 
-    process.stdout.write(`[command]${tarVersion.name} ${args.join(" ")}\n`);
+    process.stdout.write(`[command]tar ${args.join(" ")}\n`);
 
     await new Promise<void>((resolve, reject) => {
-      const tarProcess = spawn(tarVersion.name, args, { stdio: "pipe" });
+      const tarProcess = spawn("tar", args, { stdio: "pipe" });
 
       let stdout = "";
       tarProcess.stdout?.on("data", (data: Buffer) => {
@@ -221,7 +205,7 @@ export async function extractTarZst(
         if (code !== 0) {
           reject(
             new CommandInvocationError(
-              tarVersion.name,
+              "tar",
               args,
               code ?? undefined,
               stdout,

src/upload-lib.ts

@@ -14,6 +14,7 @@ import * as api from "./api-client";
 import { getGitHubVersion, wrapApiConfigurationError } from "./api-client";
 import { CodeQL, getCodeQL } from "./codeql";
 import { getConfig } from "./config-utils";
+import { readDiffRangesJsonFile } from "./diff-filtering-utils";
 import { EnvVar } from "./environment";
 import { FeatureEnablement } from "./feature-flags";
 import * as fingerprints from "./fingerprints";
@@ -578,6 +579,7 @@ export async function uploadFiles(
     features,
     logger,
   );
+  sarif = filterAlertsByDiffRange(logger, sarif);
   sarif = await fingerprints.addFingerprints(sarif, checkoutPath, logger);
 
   const analysisKey = await api.getAnalysisKey();
@@ -848,3 +850,50 @@ export class InvalidSarifUploadError extends Error {
     super(message);
   }
 }
+
+function filterAlertsByDiffRange(logger: Logger, sarif: SarifFile): SarifFile {
+  const diffRanges = readDiffRangesJsonFile(logger);
+  if (!diffRanges?.length) {
+    return sarif;
+  }
+
+  const checkoutPath = actionsUtil.getRequiredInput("checkout_path");
+
+  for (const run of sarif.runs) {
+    if (run.results) {
+      run.results = run.results.filter((result) => {
+        const locations = [
+          ...(result.locations || []).map((loc) => loc.physicalLocation),
+          ...(result.relatedLocations || []).map((loc) => loc.physicalLocation),
+        ];
+
+        return locations.some((physicalLocation) => {
+          const locationUri = physicalLocation?.artifactLocation?.uri;
+          const locationStartLine = physicalLocation?.region?.startLine;
+          if (!locationUri || locationStartLine === undefined) {
+            return false;
+          }
+          // CodeQL always uses forward slashes as the path separator, so on Windows we
+          // need to replace any backslashes with forward slashes.
+          const locationPath = path
+            .join(checkoutPath, locationUri)
+            .replaceAll(path.sep, "/");
+          // Alert filtering here replicates the same behavior as the restrictAlertsTo
+          // extensible predicate in CodeQL. See the restrictAlertsTo documentation
+          // https://codeql.github.com/codeql-standard-libraries/csharp/codeql/util/AlertFiltering.qll/predicate.AlertFiltering$restrictAlertsTo.3.html
+          // for more details, such as why the filtering applies only to the first line
+          // of an alert location.
+          return diffRanges.some(
+            (range) =>
+              range.path === locationPath &&
+              ((range.startLine <= locationStartLine &&
+                range.endLine >= locationStartLine) ||
+                (range.startLine === 0 && range.endLine === 0)),
+          );
+        });
+      });
+    }
+  }
+
+  return sarif;
+}

src/util.ts

@@ -96,6 +96,16 @@ export interface SarifResult {
       };
     };
   }>;
+  relatedLocations?: Array<{
+    physicalLocation: {
+      artifactLocation: {
+        uri: string;
+      };
+      region?: {
+        startLine?: number;
+      };
+    };
+  }>;
   partialFingerprints: {
     primaryLocationLineHash?: string;
   };