Add expect-error input to force PR check green on expected failure (#1177)

.github/workflows/debug-artifacts-failure.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest]
-    name: Failure Expected - Upload debug artifacts
+    name: Upload debug artifacts after failure in analyze
     continue-on-error: true
     timeout-minutes: 45
     runs-on: ${{ matrix.os }}
@@ -45,6 +45,7 @@ jobs:
       - uses: ./../action/analyze
         id: analysis  
         with:
+          expect-error: true
           ram: 1
   download-and-check-artifacts:
     name: Download and check debug artifacts after failure in analyze

analyze/action.yml

@@ -66,6 +66,10 @@ inputs:
     default: ${{ github.token }}
   matrix:
     default: ${{ toJson(matrix) }}
+  expect-error:
+    description: "[Internal] It is an error to use this input outside of integration testing of the codeql-action."
+    required: false
+    default: "false"
 outputs:
   db-locations:
     description: A map from language to absolute path for each database created by CodeQL.

lib/actions-util.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.sendStatusReport = exports.createStatusReportBase = exports.getActionsStatus = exports.getRef = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRunID = exports.getWorkflow = exports.formatWorkflowCause = exports.formatWorkflowErrors = exports.validateWorkflow = exports.getWorkflowErrors = exports.WorkflowErrors = exports.patternIsSuperset = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
+exports.isAnalyzingCodeQLActionRepoOrFork = exports.printDebugLogs = exports.isAnalyzingDefaultBranch = exports.getRelativeScriptPath = exports.isRunningLocalAction = exports.sendStatusReport = exports.createStatusReportBase = exports.getActionsStatus = exports.getRef = exports.computeAutomationID = exports.getAutomationID = exports.getAnalysisKey = exports.getWorkflowRunID = exports.getWorkflow = exports.formatWorkflowCause = exports.formatWorkflowErrors = exports.validateWorkflow = exports.getWorkflowErrors = exports.WorkflowErrors = exports.patternIsSuperset = exports.determineMergeBaseCommitOid = exports.getCommitOid = exports.getTemporaryDirectory = exports.getOptionalInput = exports.getRequiredInput = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
@@ -28,6 +28,7 @@ const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
 const safeWhich = __importStar(require("@chrisgavin/safe-which"));
 const yaml = __importStar(require("js-yaml"));
 const api = __importStar(require("./api-client"));
+const codeql_1 = require("./codeql");
 const sharedEnv = __importStar(require("./shared-environment"));
 const util_1 = require("./util");
 // eslint-disable-next-line import/no-commonjs
@@ -702,4 +703,19 @@ async function printDebugLogs(config) {
     }
 }
 exports.printDebugLogs = printDebugLogs;
+// Returns whether workflow kicked off by codeql-action repo itself,
+// or a fork of it.
+function isAnalyzingCodeQLActionRepoOrFork() {
+    var _a, _b;
+    const codeQLActionRepoUrl = `https://api.github.com/repos/${codeql_1.CODEQL_DEFAULT_ACTION_REPOSITORY}`;
+    const repo = (_a = getWorkflowEvent()) === null || _a === void 0 ? void 0 : _a.repository;
+    if ((repo === null || repo === void 0 ? void 0 : repo.url) === codeQLActionRepoUrl) {
+        return true;
+    }
+    if ((repo === null || repo === void 0 ? void 0 : repo.fork) && ((_b = repo === null || repo === void 0 ? void 0 : repo.parent) === null || _b === void 0 ? void 0 : _b.url) === codeQLActionRepoUrl) {
+        return true;
+    }
+    return false;
+}
+exports.isAnalyzingCodeQLActionRepoOrFork = isAnalyzingCodeQLActionRepoOrFork;
 //# sourceMappingURL=actions-util.js.map

lib/analyze-action.js

@@ -60,6 +60,12 @@ async function sendStatusReport(startedAt, config, stats, error, trapCacheUpload
     }
 }
 exports.sendStatusReport = sendStatusReport;
+// `expect-error` should only be set to any value by the
+// codeql-action repo or a fork of it.
+function hasBadExpectErrorInput() {
+    return (actionsUtil.getOptionalInput("expect-error") !== "false" &&
+        !actionsUtil.isAnalyzingCodeQLActionRepoOrFork());
+}
 async function run() {
     const startedAt = new Date();
     let uploadResult = undefined;
@@ -78,6 +84,9 @@ async function run() {
         if (config === undefined) {
             throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
         }
+        if (hasBadExpectErrorInput()) {
+            throw new Error("`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork.");
+        }
         await util.enrichEnvironment(util.Mode.actions, await (0, codeql_1.getCodeQL)(config.codeQLCmd));
         const apiDetails = {
             auth: actionsUtil.getRequiredInput("token"),
@@ -124,10 +133,17 @@ async function run() {
             actionsUtil.getRequiredInput("wait-for-processing") === "true") {
             await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());
         }
+        // If we did not throw an error yet here, but we expect one, throw it.
+        if (actionsUtil.getOptionalInput("expect-error") === "true") {
+            core.setFailed(`expect-error input was set to true but no error was thrown.`);
+        }
     }
     catch (origError) {
         const error = origError instanceof Error ? origError : new Error(String(origError));
-        core.setFailed(error.message);
+        if (actionsUtil.getOptionalInput("expect-error") !== "true" ||
+            hasBadExpectErrorInput()) {
+            core.setFailed(error.message);
+        }
         console.log(error);
         if (error instanceof analyze_1.CodeQLAnalysisError) {
             const stats = { ...error.queriesStatusReport };

lib/codeql.js

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_CONFIG_FILES = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
+exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_CONFIG_FILES = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = exports.CommandInvocationError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -56,7 +56,7 @@ exports.CommandInvocationError = CommandInvocationError;
  */
 let cachedCodeQL = undefined;
 const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
-const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
+exports.CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
 /**
  * The oldest version of CodeQL that the Action will run with. This should be
  * at least three minor versions behind the current version. The version flags
@@ -121,7 +121,7 @@ function getCodeQLBundleName() {
 }
 function getCodeQLActionRepository(logger) {
     if (!util.isActions()) {
-        return CODEQL_DEFAULT_ACTION_REPOSITORY;
+        return exports.CODEQL_DEFAULT_ACTION_REPOSITORY;
     }
     else {
         return getActionsCodeQLActionRepository(logger);
@@ -138,7 +138,7 @@ function getActionsCodeQLActionRepository(logger) {
         // This handles the case where the Action does not come from an Action repository,
         // e.g. our integration tests which use the Action code from the current checkout.
         logger.info("The CodeQL Action is checked out locally. Using the default CodeQL Action repository.");
-        return CODEQL_DEFAULT_ACTION_REPOSITORY;
+        return exports.CODEQL_DEFAULT_ACTION_REPOSITORY;
     }
     logger.info("GITHUB_ACTION_REPOSITORY environment variable was not set. Falling back to legacy method of finding the GitHub Action.");
     const relativeScriptPathParts = (0, actions_util_1.getRelativeScriptPath)().split(path.sep);
@@ -150,9 +150,9 @@ async function getCodeQLBundleDownloadURL(apiDetails, variant, logger) {
         // This GitHub instance, and this Action.
         [apiDetails.url, codeQLActionRepository],
         // This GitHub instance, and the canonical Action.
-        [apiDetails.url, CODEQL_DEFAULT_ACTION_REPOSITORY],
+        [apiDetails.url, exports.CODEQL_DEFAULT_ACTION_REPOSITORY],
         // GitHub.com, and the canonical Action.
-        [util.GITHUB_DOTCOM_URL, CODEQL_DEFAULT_ACTION_REPOSITORY],
+        [util.GITHUB_DOTCOM_URL, exports.CODEQL_DEFAULT_ACTION_REPOSITORY],
     ];
     // We now filter out any duplicates.
     // Duplicates will happen either because the GitHub instance is GitHub.com, or because the Action is not a fork.
@@ -188,7 +188,7 @@ async function getCodeQLBundleDownloadURL(apiDetails, variant, logger) {
         const [apiURL, repository] = downloadSource;
         // If we've reached the final case, short-circuit the API check since we know the bundle exists and is public.
         if (apiURL === util.GITHUB_DOTCOM_URL &&
-            repository === CODEQL_DEFAULT_ACTION_REPOSITORY) {
+            repository === exports.CODEQL_DEFAULT_ACTION_REPOSITORY) {
             break;
         }
         const [repositoryOwner, repositoryName] = repository.split("/");
@@ -209,7 +209,7 @@ async function getCodeQLBundleDownloadURL(apiDetails, variant, logger) {
             logger.info(`Looked for CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} but got error ${e}.`);
         }
     }
-    return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
+    return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${CODEQL_BUNDLE_VERSION}/${codeQLBundleName}`;
 }
 /**
  * Set up CodeQL CLI access.

src/actions-util.ts

@@ -8,6 +8,7 @@ import * as safeWhich from "@chrisgavin/safe-which";
 import * as yaml from "js-yaml";
 
 import * as api from "./api-client";
+import { CODEQL_DEFAULT_ACTION_REPOSITORY } from "./codeql";
 import { Config } from "./config-utils";
 import * as sharedEnv from "./shared-environment";
 import {
@@ -900,3 +901,17 @@ export async function printDebugLogs(config: Config) {
     walkLogFiles(logsDirectory);
   }
 }
+
+// Returns whether workflow kicked off by codeql-action repo itself,
+// or a fork of it.
+export function isAnalyzingCodeQLActionRepoOrFork(): boolean {
+  const codeQLActionRepoUrl = `https://api.github.com/repos/${CODEQL_DEFAULT_ACTION_REPOSITORY}`;
+  const repo = getWorkflowEvent()?.repository;
+  if (repo?.url === codeQLActionRepoUrl) {
+    return true;
+  }
+  if (repo?.fork && repo?.parent?.url === codeQLActionRepoUrl) {
+    return true;
+  }
+  return false;
+}

src/analyze-action.ts

@@ -83,6 +83,15 @@ export async function sendStatusReport(
   }
 }
 
+// `expect-error` should only be set to any value by the
+// codeql-action repo or a fork of it.
+function hasBadExpectErrorInput(): boolean {
+  return (
+    actionsUtil.getOptionalInput("expect-error") !== "false" &&
+    !actionsUtil.isAnalyzingCodeQLActionRepoOrFork()
+  );
+}
+
 async function run() {
   const startedAt = new Date();
   let uploadResult: UploadResult | undefined = undefined;
@@ -112,6 +121,13 @@ async function run() {
         "Config file could not be found at expected location. Has the 'init' action been called?"
       );
     }
+
+    if (hasBadExpectErrorInput()) {
+      throw new Error(
+        "`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork."
+      );
+    }
+
     await util.enrichEnvironment(
       util.Mode.actions,
       await getCodeQL(config.codeQLCmd)
@@ -206,10 +222,22 @@ async function run() {
         getActionsLogger()
       );
     }
+    // If we did not throw an error yet here, but we expect one, throw it.
+    if (actionsUtil.getOptionalInput("expect-error") === "true") {
+      core.setFailed(
+        `expect-error input was set to true but no error was thrown.`
+      );
+    }
   } catch (origError) {
     const error =
       origError instanceof Error ? origError : new Error(String(origError));
-    core.setFailed(error.message);
+    if (
+      actionsUtil.getOptionalInput("expect-error") !== "true" ||
+      hasBadExpectErrorInput()
+    ) {
+      core.setFailed(error.message);
+    }
+
     console.log(error);
 
     if (error instanceof CodeQLAnalysisError) {

src/codeql.ts

@@ -222,7 +222,7 @@ interface PackDownloadItem {
 let cachedCodeQL: CodeQL | undefined = undefined;
 
 const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
-const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
+export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
 
 /**
  * The oldest version of CodeQL that the Action will run with. This should be