Use github merge-results command when feature flag is enabled

lib/codeql.js

@@ -207,6 +207,7 @@ function setCodeQL(partialCodeql) {
         databaseExportDiagnostics: resolveFunction(partialCodeql, "databaseExportDiagnostics"),
         diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
         resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
+        mergeResults: resolveFunction(partialCodeql, "mergeResults"),
     };
     return cachedCodeQL;
 }
@@ -664,6 +665,22 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             }).exec();
             return JSON.parse(extractorPath);
         },
+        async mergeResults(sarifFiles, outputFile, mergeRunsFromEqualCategory = false) {
+            const args = [
+                "github",
+                "merge-results",
+                "--output",
+                outputFile,
+                ...getExtraOptionsFromEnv(["github", "merge-results"]),
+            ];
+            for (const sarifFile of sarifFiles) {
+                args.push("--sarif", sarifFile);
+            }
+            if (mergeRunsFromEqualCategory) {
+                args.push("--sarif-merge-runs-from-equal-category");
+            }
+            await new toolrunner.ToolRunner(cmd, args).exec();
+        },
     };
     // To ensure that status reports include the CodeQL CLI version wherever
     // possible, we want to call getVersion(), which populates the version value

lib/upload-lib.js

@@ -34,9 +34,15 @@ const core = __importStar(require("@actions/core"));
 const file_url_1 = __importDefault(require("file-url"));
 const jsonschema = __importStar(require("jsonschema"));
 const actionsUtil = __importStar(require("./actions-util"));
+const actions_util_1 = require("./actions-util");
 const api = __importStar(require("./api-client"));
+const api_client_1 = require("./api-client");
+const codeql_1 = require("./codeql");
+const config_utils_1 = require("./config-utils");
 const environment_1 = require("./environment");
+const feature_flags_1 = require("./feature-flags");
 const fingerprints = __importStar(require("./fingerprints"));
+const init_1 = require("./init");
 const repository_1 = require("./repository");
 const util = __importStar(require("./util"));
 const util_1 = require("./util");
@@ -62,6 +68,53 @@ function combineSarifFiles(sarifFiles) {
     }
     return combinedSarif;
 }
+// Takes a list of paths to sarif files and combines them together using the
+// CLI `github merge-results` command when all SARIF files are produced by
+// CodeQL. Otherwise, it will fall back to combining the files in the action.
+// Returns the contents of the combined sarif file.
+async function combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, logger) {
+    // First check if all files are produced by CodeQL.
+    let allCodeQL = true;
+    for (const sarifFile of sarifFiles) {
+        const sarifObject = JSON.parse(fs.readFileSync(sarifFile, "utf8"));
+        const allRunsCodeQL = sarifObject.runs?.every((run) => run.tool?.driver?.name === "CodeQL");
+        if (!allRunsCodeQL) {
+            allCodeQL = false;
+            break;
+        }
+    }
+    if (!allCodeQL) {
+        logger.warning("Not all SARIF files were produced by CodeQL. Merging files in the action.");
+        // If not, use the naive method of combining the files.
+        return combineSarifFiles(sarifFiles);
+    }
+    // Initialize CodeQL, either by using the config file from the 'init' step,
+    // or by initializing it here.
+    let codeQL;
+    let tempDir;
+    const config = await (0, config_utils_1.getConfig)(actionsUtil.getTemporaryDirectory(), logger);
+    if (config !== undefined) {
+        codeQL = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+        tempDir = config.tempDir;
+    }
+    else {
+        logger.warning("Initializing CodeQL since the 'init' Action was not called before this step.");
+        const apiDetails = {
+            auth: (0, actions_util_1.getRequiredInput)("token"),
+            externalRepoAuth: (0, actions_util_1.getOptionalInput)("external-repository-token"),
+            url: (0, util_1.getRequiredEnvParam)("GITHUB_SERVER_URL"),
+            apiURL: (0, util_1.getRequiredEnvParam)("GITHUB_API_URL"),
+        };
+        const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(gitHubVersion.type);
+        const initCodeQLResult = await (0, init_1.initCodeQL)(undefined, // There is no tools input on the upload action
+        apiDetails, (0, actions_util_1.getTemporaryDirectory)(), gitHubVersion.type, codeQLDefaultVersionInfo, logger);
+        codeQL = initCodeQLResult.codeql;
+        tempDir = (0, actions_util_1.getTemporaryDirectory)();
+    }
+    const outputFile = path.resolve(tempDir, "combined-sarif.sarif");
+    await codeQL.mergeResults(sarifFiles, outputFile, true);
+    return JSON.parse(fs.readFileSync(outputFile, "utf8"));
+}
 // Populates the run.automationDetails.id field using the analysis_key and environment
 // and return an updated sarif file contents.
 function populateRunAutomationDetails(sarif, category, analysis_key, environment) {
@@ -264,11 +317,15 @@ exports.buildPayload = buildPayload;
 async function uploadFiles(sarifFiles, repositoryNwo, commitOid, ref, analysisKey, category, analysisName, workflowRunID, workflowRunAttempt, sourceRoot, environment, logger) {
     logger.startGroup("Uploading results");
     logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);
+    const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
+    const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
     // Validate that the files we were asked to upload are all valid SARIF files
     for (const file of sarifFiles) {
         validateSarifFileSchema(file, logger);
     }
-    let sarif = combineSarifFiles(sarifFiles);
+    let sarif = (await features.getValue(feature_flags_1.Feature.CliSarifMerge))
+        ? await combineSarifFilesUsingCLI(sarifFiles, gitHubVersion, features, logger)
+        : combineSarifFiles(sarifFiles);
     sarif = await fingerprints.addFingerprints(sarif, sourceRoot, logger);
     sarif = populateRunAutomationDetails(sarif, category, analysisKey, environment);
     const toolNames = util.getToolNames(sarif);

src/codeql.ts

@@ -50,6 +50,10 @@ interface ExtraOptions {
     extractor?: Options;
     queries?: Options;
   };
+  github?: {
+    "*"?: Options;
+    "merge-results"?: Options;
+  };
 }
 
 export interface CodeQL {
@@ -191,6 +195,14 @@ export interface CodeQL {
   ): Promise<void>;
   /** Get the location of an extractor for the specified language. */
   resolveExtractor(language: Language): Promise<string>;
+  /**
+   * Run 'codeql github merge-results'.
+   */
+  mergeResults(
+    sarifFiles: string[],
+    outputFile: string,
+    mergeRunsFromEqualCategory?: boolean,
+  ): Promise<void>;
 }
 
 export interface VersionInfo {
@@ -489,6 +501,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     ),
     diagnosticsExport: resolveFunction(partialCodeql, "diagnosticsExport"),
     resolveExtractor: resolveFunction(partialCodeql, "resolveExtractor"),
+    mergeResults: resolveFunction(partialCodeql, "mergeResults"),
   };
   return cachedCodeQL;
 }
@@ -1077,6 +1090,29 @@ export async function getCodeQLForCmd(
       ).exec();
       return JSON.parse(extractorPath);
     },
+    async mergeResults(
+      sarifFiles: string[],
+      outputFile: string,
+      mergeRunsFromEqualCategory = false,
+    ): Promise<void> {
+      const args = [
+        "github",
+        "merge-results",
+        "--output",
+        outputFile,
+        ...getExtraOptionsFromEnv(["github", "merge-results"]),
+      ];
+
+      for (const sarifFile of sarifFiles) {
+        args.push("--sarif", sarifFile);
+      }
+
+      if (mergeRunsFromEqualCategory) {
+        args.push("--sarif-merge-runs-from-equal-category");
+      }
+
+      await new toolrunner.ToolRunner(cmd, args).exec();
+    },
   };
   // To ensure that status reports include the CodeQL CLI version wherever
   // possible, we want to call getVersion(), which populates the version value

src/upload-lib.ts

@@ -8,13 +8,29 @@ import fileUrl from "file-url";
 import * as jsonschema from "jsonschema";
 
 import * as actionsUtil from "./actions-util";
+import {
+  getOptionalInput,
+  getRequiredInput,
+  getTemporaryDirectory,
+} from "./actions-util";
 import * as api from "./api-client";
+import { getGitHubVersion } from "./api-client";
+import { CodeQL, getCodeQL } from "./codeql";
+import { getConfig } from "./config-utils";
 import { EnvVar } from "./environment";
+import { Feature, Features } from "./feature-flags";
 import * as fingerprints from "./fingerprints";
+import { initCodeQL } from "./init";
 import { Logger } from "./logging";
 import { parseRepositoryNwo, RepositoryNwo } from "./repository";
 import * as util from "./util";
-import { SarifFile, ConfigurationError, wrapError } from "./util";
+import {
+  SarifFile,
+  ConfigurationError,
+  wrapError,
+  getRequiredEnvParam,
+  GitHubVersion,
+} from "./util";
 
 const GENERIC_403_MSG =
   "The repo on which this action is running has not opted-in to CodeQL code scanning.";
@@ -48,6 +64,88 @@ function combineSarifFiles(sarifFiles: string[]): SarifFile {
   return combinedSarif;
 }
 
+// Takes a list of paths to sarif files and combines them together using the
+// CLI `github merge-results` command when all SARIF files are produced by
+// CodeQL. Otherwise, it will fall back to combining the files in the action.
+// Returns the contents of the combined sarif file.
+async function combineSarifFilesUsingCLI(
+  sarifFiles: string[],
+  gitHubVersion: GitHubVersion,
+  features: Features,
+  logger: Logger,
+): Promise<SarifFile> {
+  // First check if all files are produced by CodeQL.
+  let allCodeQL = true;
+
+  for (const sarifFile of sarifFiles) {
+    const sarifObject = JSON.parse(
+      fs.readFileSync(sarifFile, "utf8"),
+    ) as SarifFile;
+
+    const allRunsCodeQL = sarifObject.runs?.every(
+      (run) => run.tool?.driver?.name === "CodeQL",
+    );
+
+    if (!allRunsCodeQL) {
+      allCodeQL = false;
+      break;
+    }
+  }
+
+  if (!allCodeQL) {
+    logger.warning(
+      "Not all SARIF files were produced by CodeQL. Merging files in the action.",
+    );
+
+    // If not, use the naive method of combining the files.
+    return combineSarifFiles(sarifFiles);
+  }
+
+  // Initialize CodeQL, either by using the config file from the 'init' step,
+  // or by initializing it here.
+  let codeQL: CodeQL;
+  let tempDir: string;
+
+  const config = await getConfig(actionsUtil.getTemporaryDirectory(), logger);
+  if (config !== undefined) {
+    codeQL = await getCodeQL(config.codeQLCmd);
+    tempDir = config.tempDir;
+  } else {
+    logger.warning(
+      "Initializing CodeQL since the 'init' Action was not called before this step.",
+    );
+
+    const apiDetails = {
+      auth: getRequiredInput("token"),
+      externalRepoAuth: getOptionalInput("external-repository-token"),
+      url: getRequiredEnvParam("GITHUB_SERVER_URL"),
+      apiURL: getRequiredEnvParam("GITHUB_API_URL"),
+    };
+
+    const codeQLDefaultVersionInfo = await features.getDefaultCliVersion(
+      gitHubVersion.type,
+    );
+
+    const initCodeQLResult = await initCodeQL(
+      undefined, // There is no tools input on the upload action
+      apiDetails,
+      getTemporaryDirectory(),
+      gitHubVersion.type,
+      codeQLDefaultVersionInfo,
+      logger,
+    );
+
+    codeQL = initCodeQLResult.codeql;
+    tempDir = getTemporaryDirectory();
+  }
+
+  const outputFile = path.resolve(tempDir, "combined-sarif.sarif");
+
+  await codeQL.mergeResults(sarifFiles, outputFile, true);
+
+  return JSON.parse(fs.readFileSync(outputFile, "utf8")) as SarifFile;
+}
+
 // Populates the run.automationDetails.id field using the analysis_key and environment
 // and return an updated sarif file contents.
 export function populateRunAutomationDetails(
@@ -363,12 +461,27 @@ async function uploadFiles(
   logger.startGroup("Uploading results");
   logger.info(`Processing sarif files: ${JSON.stringify(sarifFiles)}`);
 
+  const gitHubVersion = await getGitHubVersion();
+  const features = new Features(
+    gitHubVersion,
+    repositoryNwo,
+    actionsUtil.getTemporaryDirectory(),
+    logger,
+  );
+
   // Validate that the files we were asked to upload are all valid SARIF files
   for (const file of sarifFiles) {
     validateSarifFileSchema(file, logger);
   }
 
-  let sarif = combineSarifFiles(sarifFiles);
+  let sarif = (await features.getValue(Feature.CliSarifMerge))
+    ? await combineSarifFilesUsingCLI(
+        sarifFiles,
+        gitHubVersion,
+        features,
+        logger,
+      )
+    : combineSarifFiles(sarifFiles);
   sarif = await fingerprints.addFingerprints(sarif, sourceRoot, logger);
 
   sarif = populateRunAutomationDetails(