Merge branch 'main' into igfoo/code_scanning_codeql_java_lombok

lib/actions-util.js

@@ -39,7 +39,11 @@ const pkg = require("../package.json");
  * This allows us to get stronger type checking of required/optional inputs.
  */
 const getRequiredInput = function (name) {
-    return core.getInput(name, { required: true });
+    const value = core.getInput(name);
+    if (!value) {
+        throw new util_1.UserError(`Input required and not supplied: ${name}`);
+    }
+    return value;
 };
 exports.getRequiredInput = getRequiredInput;
 /**

lib/analyze.js

@@ -232,7 +232,9 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
                 }
                 statusReport["event_reports"].push(perQueryAlertCountEventReport);
             }
-            await runPrintLinesOfCode(language);
+            if (!(await features.getValue(feature_flags_1.Feature.NewAnalysisSummaryEnabled, codeql))) {
+                await runPrintLinesOfCode(language);
+            }
         }
         catch (e) {
             logger.info(String(e));
@@ -283,7 +285,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
             fs.writeFileSync(querySuitePath, querySuiteContents);
             logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
         }
-        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun);
+        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun, features);
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
         return querySuitePath;
     }
@@ -297,7 +299,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         const querySuitePath = `${databasePath}-queries-${type}.qls`;
         fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
-        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun);
+        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun, features);
         return querySuitePath;
     }
 }

lib/codeql.js

@@ -419,7 +419,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 throw new Error(`Unexpected output from codeql resolve build-environment: ${e} in\n${output}`);
             }
         },
-        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun) {
+        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun, features) {
             const codeqlArgs = [
                 "database",
                 "run-queries",
@@ -439,6 +439,12 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             if (querySuitePath) {
                 codeqlArgs.push(querySuitePath);
             }
+            if (await features.getValue(feature_flags_1.Feature.EvaluatorIntraLayerParallelismEnabled, this)) {
+                codeqlArgs.push("--intra-layer-parallelism");
+            }
+            else if (await util.codeQlVersionAbove(this, feature_flags_1.CODEQL_VERSION_INTRA_LAYER_PARALLELISM)) {
+                codeqlArgs.push("--no-intra-layer-parallelism");
+            }
             await runTool(cmd, codeqlArgs);
         },
         async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, verbosityFlag, automationDetailsId, config, features, logger) {
@@ -485,12 +491,15 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             if (querySuitePaths) {
                 codeqlArgs.push(...querySuitePaths);
             }
-            // capture stdout, which contains analysis summaries
-            const returnState = await runTool(cmd, codeqlArgs);
+            // Capture the stdout, which contains the analysis summary. Don't stream it to the Actions
+            // logs to avoid printing it twice.
+            const analysisSummary = await runTool(cmd, codeqlArgs, {
+                noStreamStdout: true,
+            });
             if (shouldWorkaroundInvalidNotifications) {
                 util.fixInvalidNotificationsInFile(codeqlOutputFile, sarifFile, logger);
             }
-            return returnState;
+            return analysisSummary;
         },
         async databasePrintBaseline(databasePath) {
             const codeqlArgs = [
@@ -718,9 +727,13 @@ async function runTool(cmd, args = [], opts = {}) {
     let output = "";
     let error = "";
     const exitCode = await new toolrunner.ToolRunner(cmd, args, {
+        ignoreReturnCode: true,
         listeners: {
             stdout: (data) => {
                 output += data.toString("utf8");
+                if (!opts.noStreamStdout) {
+                    process.stdout.write(data);
+                }
             },
             stderr: (data) => {
                 let readStartIndex = 0;
@@ -730,9 +743,11 @@ async function runTool(cmd, args = [], opts = {}) {
                     readStartIndex = data.length - maxErrorSize + 1;
                 }
                 error += data.toString("utf8", readStartIndex);
+                // Mimic the standard behavior of the toolrunner by writing stderr to stdout
+                process.stdout.write(data);
             },
         },
-        ignoreReturnCode: true,
+        silent: true,
         ...(opts.stdin ? { input: Buffer.from(opts.stdin || "") } : {}),
     }).exec();
     if (exitCode !== 0) {

lib/defaults.json

@@ -1,6 +1,6 @@
 {
-    "bundleVersion": "codeql-bundle-v2.14.0",
-    "cliVersion": "2.14.0",
-    "priorBundleVersion": "codeql-bundle-v2.13.5",
-    "priorCliVersion": "2.13.5"
+    "bundleVersion": "codeql-bundle-v2.14.1",
+    "cliVersion": "2.14.1",
+    "priorBundleVersion": "codeql-bundle-v2.14.0",
+    "priorCliVersion": "2.14.0"
 }

lib/feature-flags.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_NEW_ANALYSIS_SUMMARY = exports.CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = void 0;
+exports.logCodeScanningConfigInCli = exports.useCodeScanningConfigInCli = exports.Features = exports.FEATURE_FLAGS_FILE_NAME = exports.featureConfig = exports.Feature = exports.CODEQL_VERSION_INTRA_LAYER_PARALLELISM = exports.CODEQL_VERSION_NEW_ANALYSIS_SUMMARY = exports.CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const semver = __importStar(require("semver"));
@@ -40,6 +40,10 @@ exports.CODEQL_VERSION_BUNDLE_SEMANTICALLY_VERSIONED = "2.13.4";
  * Versions 2.14.0+ of the CodeQL CLI support new analysis summaries.
  */
 exports.CODEQL_VERSION_NEW_ANALYSIS_SUMMARY = "2.14.0";
+/**
+ * Versions 2.14.0+ of the CodeQL CLI support intra-layer parallelism (aka fine-grained parallelism) options.
+ */
+exports.CODEQL_VERSION_INTRA_LAYER_PARALLELISM = "2.14.0";
 /**
  * Feature enablement as returned by the GitHub API endpoint.
  *
@@ -51,6 +55,7 @@ var Feature;
     Feature["CodeqlJavaLombokEnabled"] = "codeql_java_lombok_enabled";
     Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
     Feature["DisablePythonDependencyInstallationEnabled"] = "disable_python_dependency_installation_enabled";
+    Feature["EvaluatorIntraLayerParallelismEnabled"] = "evaluator_intra_layer_parallelism_enabled";
     Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     Feature["NewAnalysisSummaryEnabled"] = "new_analysis_summary_enabled";
@@ -74,6 +79,11 @@ exports.featureConfig = {
         minimumVersion: "2.11.6",
         defaultValue: true,
     },
+    [Feature.EvaluatorIntraLayerParallelismEnabled]: {
+        envVar: "CODEQL_EVALUATOR_INTRA_LAYER_PARALLELISM",
+        minimumVersion: exports.CODEQL_VERSION_INTRA_LAYER_PARALLELISM,
+        defaultValue: false,
+    },
     [Feature.ExportDiagnosticsEnabled]: {
         envVar: "CODEQL_ACTION_EXPORT_DIAGNOSTICS",
         minimumVersion: "2.12.4",

lib/setup-codeql.js

@@ -31,6 +31,7 @@ const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const perf_hooks_1 = require("perf_hooks");
 const toolcache = __importStar(require("@actions/tool-cache"));
+const del_1 = __importDefault(require("del"));
 const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
 const semver = __importStar(require("semver"));
 const uuid_1 = require("uuid");
@@ -420,17 +421,22 @@ async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, ap
     const dest = path.join(tempDir, (0, uuid_1.v4)());
     const finalHeaders = Object.assign({ "User-Agent": "CodeQL Action" }, headers);
     const toolsDownloadStart = perf_hooks_1.performance.now();
-    const codeqlPath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
+    const archivedBundlePath = await toolcache.downloadTool(codeqlURL, dest, authorization, finalHeaders);
     const toolsDownloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
-    logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-    const codeqlExtracted = await toolcache.extractTar(codeqlPath);
+    logger.debug(`Finished downloading CodeQL bundle to ${archivedBundlePath} (${toolsDownloadDurationMs} ms).`);
+    logger.debug("Extracting CodeQL bundle.");
+    const extractionStart = perf_hooks_1.performance.now();
+    const extractedBundlePath = await toolcache.extractTar(archivedBundlePath);
+    const extractionMs = Math.round(perf_hooks_1.performance.now() - extractionStart);
+    logger.debug(`Finished extracting CodeQL bundle to ${extractedBundlePath} (${extractionMs} ms).`);
+    await cleanUpGlob(archivedBundlePath, "CodeQL bundle archive", logger);
     const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
     if (bundleVersion === undefined) {
         logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
             `URL ${codeqlURL}.`);
         return {
             toolsVersion: maybeCliVersion ?? "unknown",
-            codeqlFolder: codeqlExtracted,
+            codeqlFolder: extractedBundlePath,
             toolsDownloadDurationMs,
         };
     }
@@ -452,9 +458,15 @@ async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, ap
     const toolcacheVersion = maybeCliVersion?.match(/^[0-9]+\.[0-9]+\.[0-9]+$/)
         ? `${maybeCliVersion}-${bundleVersion}`
         : convertToSemVer(bundleVersion, logger);
+    logger.debug("Caching CodeQL bundle.");
+    const toolcachedBundlePath = await toolcache.cacheDir(extractedBundlePath, "CodeQL", toolcacheVersion);
+    // Defensive check: we expect `cacheDir` to copy the bundle to a new location.
+    if (toolcachedBundlePath !== extractedBundlePath) {
+        await cleanUpGlob(extractedBundlePath, "CodeQL bundle from temporary directory", logger);
+    }
     return {
         toolsVersion: maybeCliVersion ?? toolcacheVersion,
-        codeqlFolder: await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion),
+        codeqlFolder: toolcachedBundlePath,
         toolsDownloadDurationMs,
     };
 }
@@ -510,4 +522,22 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, defau
     return { codeqlFolder, toolsDownloadDurationMs, toolsSource, toolsVersion };
 }
 exports.setupCodeQLBundle = setupCodeQLBundle;
+async function cleanUpGlob(glob, name, logger) {
+    logger.debug(`Cleaning up ${name}.`);
+    try {
+        const deletedPaths = await (0, del_1.default)(glob, { force: true });
+        if (deletedPaths.length === 0) {
+            logger.warning(`Failed to clean up ${name}: no files found matching ${glob}.`);
+        }
+        else if (deletedPaths.length === 1) {
+            logger.debug(`Cleaned up ${name}.`);
+        }
+        else {
+            logger.debug(`Cleaned up ${name} (${deletedPaths.length} files).`);
+        }
+    }
+    catch (e) {
+        logger.warning(`Failed to clean up ${name}: ${e}.`);
+    }
+}
 //# sourceMappingURL=setup-codeql.js.map

lib/upload-lib.js

@@ -331,7 +331,10 @@ async function waitForProcessing(repositoryNwo, sarifID, logger, options = {
                 break;
             }
             else if (status === "failed") {
-                throw new Error(`Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`);
+                const message = `Code Scanning could not process the submitted SARIF file:\n${response.data.errors}`;
+                throw shouldConsiderAsUserError(response.data.errors)
+                    ? new util_1.UserError(message)
+                    : new Error(message);
             }
             else {
                 util.assertNever(status);
@@ -346,6 +349,14 @@ async function waitForProcessing(repositoryNwo, sarifID, logger, options = {
     }
 }
 exports.waitForProcessing = waitForProcessing;
+/**
+ * Returns whether the provided processing errors should be considered a user error.
+ */
+function shouldConsiderAsUserError(processingErrors) {
+    return (processingErrors.length === 1 &&
+        processingErrors[0] ===
+            "CodeQL analyses from advanced configurations cannot be processed when the default setup is enabled");
+}
 /**
  * Checks the processing result for an unsuccessful execution. Throws if the
  * result is not a failure with a single "unsuccessful execution" error.