Merge branch 'main' into hmakholm/pr/2.5.6

src/actions-util.test.ts

@@ -1,10 +1,13 @@
+import * as fs from "fs";
+import * as path from "path";
+
 import test from "ava";
 import * as yaml from "js-yaml";
 import sinon from "sinon";
 
 import * as actionsutil from "./actions-util";
 import { setupTests } from "./testing-utils";
-import { getMode, initializeEnvironment, Mode } from "./util";
+import { getMode, initializeEnvironment, Mode, withTmpDir } from "./util";
 
 function errorCodes(
   actual: actionsutil.CodedError[],
@@ -652,3 +655,28 @@ test("initializeEnvironment", (t) => {
   t.deepEqual(getMode(), Mode.runner);
   t.deepEqual(process.env.CODEQL_ACTION_VERSION, "4.5.6");
 });
+
+test("isAnalyzingDefaultBranch()", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    const envFile = path.join(tmpDir, "event.json");
+    fs.writeFileSync(
+      envFile,
+      JSON.stringify({
+        repository: {
+          default_branch: "main",
+        },
+      })
+    );
+    process.env["GITHUB_EVENT_PATH"] = envFile;
+
+    process.env["GITHUB_REF"] = "main";
+    process.env["GITHUB_SHA"] = "1234";
+    t.deepEqual(await actionsutil.isAnalyzingDefaultBranch(), true);
+
+    process.env["GITHUB_REF"] = "refs/heads/main";
+    t.deepEqual(await actionsutil.isAnalyzingDefaultBranch(), true);
+
+    process.env["GITHUB_REF"] = "feature";
+    t.deepEqual(await actionsutil.isAnalyzingDefaultBranch(), false);
+  });
+});

src/actions-util.ts

@@ -8,7 +8,7 @@ import * as yaml from "js-yaml";
 
 import * as api from "./api-client";
 import * as sharedEnv from "./shared-environment";
-import { getRequiredEnvParam, GITHUB_DOTCOM_URL } from "./util";
+import { getRequiredEnvParam, GITHUB_DOTCOM_URL, isHTTPError } from "./util";
 
 /**
  * The utils in this module are meant to be run inside of the action only.
@@ -576,15 +576,6 @@ export async function createStatusReportBase(
   return statusReport;
 }
 
-interface HTTPError {
-  status: number;
-  message?: string;
-}
-
-function isHTTPError(arg: any): arg is HTTPError {
-  return arg?.status !== undefined && Number.isInteger(arg.status);
-}
-
 const GENERIC_403_MSG =
   "The repo on which this action is running is not opted-in to CodeQL code scanning.";
 const GENERIC_404_MSG =
@@ -691,3 +682,30 @@ export function getRelativeScriptPath(): string {
   const actionsDirectory = path.join(path.dirname(runnerTemp), "_actions");
   return path.relative(actionsDirectory, __filename);
 }
+
+// Reads the contents of GITHUB_EVENT_PATH as a JSON object
+function getWorkflowEvent(): any {
+  const eventJsonFile = getRequiredEnvParam("GITHUB_EVENT_PATH");
+  try {
+    return JSON.parse(fs.readFileSync(eventJsonFile, "utf-8"));
+  } catch (e) {
+    throw new Error(
+      `Unable to read workflow event JSON from ${eventJsonFile}: ${e}`
+    );
+  }
+}
+
+// Is the version of the repository we are currently analyzing from the default branch,
+// or alternatively from another branch or a pull request.
+export async function isAnalyzingDefaultBranch(): Promise<boolean> {
+  // Get the current ref and trim and refs/heads/ prefix
+  let currentRef = await getRef();
+  currentRef = currentRef.startsWith("refs/heads/")
+    ? currentRef.substr("refs/heads/".length)
+    : currentRef;
+
+  const event = getWorkflowEvent();
+  const defaultBranch = event?.repository?.default_branch;
+
+  return currentRef === defaultBranch;
+}

src/analyze-action.ts

@@ -11,7 +11,9 @@ import {
   runCleanup,
 } from "./analyze";
 import { Config, getConfig } from "./config-utils";
+import { uploadDatabases } from "./database-upload";
 import { getActionsLogger } from "./logging";
+import { parseRepositoryNwo } from "./repository";
 import * as upload_lib from "./upload-lib";
 import * as util from "./util";
 
@@ -116,6 +118,11 @@ async function run() {
       logger.info("Not uploading results");
       stats = { ...queriesStats };
     }
+
+    const repositoryNwo = parseRepositoryNwo(
+      util.getRequiredEnvParam("GITHUB_REPOSITORY")
+    );
+    await uploadDatabases(repositoryNwo, config, apiDetails, logger);
   } catch (error) {
     core.setFailed(error.message);
     console.log(error);

src/codeql.ts

@@ -99,6 +99,10 @@ export interface CodeQL {
    * Run 'codeql database cleanup'.
    */
   databaseCleanup(databasePath: string, cleanupLevel: string): Promise<void>;
+  /**
+   * Run 'codeql database bundle'.
+   */
+  databaseBundle(databasePath: string, outputFilePath: string): Promise<void>;
   /**
    * Run 'codeql database run-queries'.
    */
@@ -512,6 +516,7 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
     resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
     packDownload: resolveFunction(partialCodeql, "packDownload"),
     databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
+    databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
     databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
     databaseInterpretResults: resolveFunction(
       partialCodeql,
@@ -829,6 +834,18 @@ function getCodeQLForCmd(cmd: string): CodeQL {
       ];
       await runTool(cmd, codeqlArgs);
     },
+    async databaseBundle(
+      databasePath: string,
+      outputFilePath: string
+    ): Promise<void> {
+      const args = [
+        "database",
+        "bundle",
+        databasePath,
+        `--output=${outputFilePath}`,
+      ];
+      await new toolrunner.ToolRunner(cmd, args).exec();
+    },
   };
 }
 

src/database-upload.test.ts

@@ -0,0 +1,351 @@
+import * as fs from "fs";
+
+import * as github from "@actions/github";
+import test from "ava";
+import sinon from "sinon";
+
+import * as actionsUtil from "./actions-util";
+import { GitHubApiDetails } from "./api-client";
+import * as apiClient from "./api-client";
+import { setCodeQL } from "./codeql";
+import { Config } from "./config-utils";
+import { uploadDatabases } from "./database-upload";
+import { Language } from "./languages";
+import { Logger } from "./logging";
+import { RepositoryNwo } from "./repository";
+import { setupActionsVars, setupTests } from "./testing-utils";
+import {
+  GitHubVariant,
+  HTTPError,
+  initializeEnvironment,
+  Mode,
+  withTmpDir,
+} from "./util";
+
+setupTests(test);
+
+test.beforeEach(() => {
+  initializeEnvironment(Mode.actions, "1.2.3");
+});
+
+const testRepoName: RepositoryNwo = { owner: "github", repo: "example" };
+const testApiDetails: GitHubApiDetails = {
+  auth: "1234",
+  url: "https://github.com",
+};
+
+function getTestConfig(tmpDir: string): Config {
+  return {
+    languages: [Language.javascript],
+    queries: {},
+    pathsIgnore: [],
+    paths: [],
+    originalUserInput: {},
+    tempDir: tmpDir,
+    toolCacheDir: tmpDir,
+    codeQLCmd: "foo",
+    gitHubVersion: { type: GitHubVariant.DOTCOM },
+    dbLocation: tmpDir,
+    packs: {},
+  };
+}
+
+interface LoggedMessage {
+  type: "debug" | "info" | "warning" | "error";
+  message: string;
+}
+
+function getRecordingLogger(messages: LoggedMessage[]): Logger {
+  return {
+    debug: (message: string) => {
+      messages.push({ type: "debug", message });
+      console.debug(message);
+    },
+    info: (message: string) => {
+      messages.push({ type: "info", message });
+      console.info(message);
+    },
+    warning: (message: string) => {
+      messages.push({ type: "warning", message });
+      console.warn(message);
+    },
+    error: (message: string) => {
+      messages.push({ type: "error", message });
+      console.error(message);
+    },
+    isDebug: () => true,
+    startGroup: () => undefined,
+    endGroup: () => undefined,
+  };
+}
+
+function mockHttpRequests(
+  optInStatusCode: number,
+  databaseUploadStatusCode?: number
+) {
+  // Passing an auth token is required, so we just use a dummy value
+  const client = github.getOctokit("123");
+
+  const requestSpy = sinon.stub(client, "request");
+
+  const optInSpy = requestSpy.withArgs(
+    "GET /repos/:owner/:repo/code-scanning/databases"
+  );
+  if (optInStatusCode < 300) {
+    optInSpy.resolves(undefined);
+  } else {
+    optInSpy.throws(new HTTPError("some error message", optInStatusCode));
+  }
+
+  if (databaseUploadStatusCode !== undefined) {
+    const databaseUploadSpy = requestSpy.withArgs(
+      "PUT /repos/:owner/:repo/code-scanning/databases/javascript"
+    );
+    if (databaseUploadStatusCode < 300) {
+      databaseUploadSpy.resolves(undefined);
+    } else {
+      databaseUploadSpy.throws(
+        new HTTPError("some error message", databaseUploadStatusCode)
+      );
+    }
+  }
+
+  sinon.stub(apiClient, "getApiClient").value(() => client);
+}
+
+test("Abort database upload if 'upload-database' input set to false", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("false");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    const loggedMessages = [];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v: LoggedMessage) =>
+          v.type === "debug" &&
+          v.message === "Database upload disabled in workflow. Skipping upload."
+      ) !== undefined
+    );
+  });
+});
+
+test("Abort database upload if running against GHES", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    const config = getTestConfig(tmpDir);
+    config.gitHubVersion = { type: GitHubVariant.GHES, version: "3.0" };
+
+    const loggedMessages = [];
+    await uploadDatabases(
+      testRepoName,
+      config,
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v: LoggedMessage) =>
+          v.type === "debug" &&
+          v.message === "Not running against github.com. Skipping upload."
+      ) !== undefined
+    );
+  });
+});
+
+test("Abort database upload if running against GHAE", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    const config = getTestConfig(tmpDir);
+    config.gitHubVersion = { type: GitHubVariant.GHAE };
+
+    const loggedMessages = [];
+    await uploadDatabases(
+      testRepoName,
+      config,
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v: LoggedMessage) =>
+          v.type === "debug" &&
+          v.message === "Not running against github.com. Skipping upload."
+      ) !== undefined
+    );
+  });
+});
+
+test("Abort database upload if not analyzing default branch", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
+
+    const loggedMessages = [];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v: LoggedMessage) =>
+          v.type === "debug" &&
+          v.message === "Not analyzing default branch. Skipping upload."
+      ) !== undefined
+    );
+  });
+});
+
+test("Abort database upload if opt-in request returns 404", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    mockHttpRequests(404);
+
+    const loggedMessages = [];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v: LoggedMessage) =>
+          v.type === "debug" &&
+          v.message ===
+            "Repository is not opted in to database uploads. Skipping upload."
+      ) !== undefined
+    );
+  });
+});
+
+test("Abort database upload if opt-in request fails with something other than 404", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    mockHttpRequests(500);
+
+    const loggedMessages = [] as LoggedMessage[];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v) =>
+          v.type === "info" &&
+          v.message ===
+            "Skipping database upload due to unknown error: Error: some error message"
+      ) !== undefined
+    );
+  });
+});
+
+test("Don't crash if uploading a database fails", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    mockHttpRequests(204, 500);
+
+    setCodeQL({
+      async databaseBundle(_: string, outputFilePath: string) {
+        fs.writeFileSync(outputFilePath, "");
+      },
+    });
+
+    const loggedMessages = [] as LoggedMessage[];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v) =>
+          v.type === "warning" &&
+          v.message ===
+            "Failed to upload database for javascript: Error: some error message"
+      ) !== undefined
+    );
+  });
+});
+
+test("Successfully uploading a database", async (t) => {
+  await withTmpDir(async (tmpDir) => {
+    setupActionsVars(tmpDir, tmpDir);
+    sinon
+      .stub(actionsUtil, "getRequiredInput")
+      .withArgs("upload-database")
+      .returns("true");
+    sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
+
+    mockHttpRequests(204, 201);
+
+    setCodeQL({
+      async databaseBundle(_: string, outputFilePath: string) {
+        fs.writeFileSync(outputFilePath, "");
+      },
+    });
+
+    const loggedMessages = [] as LoggedMessage[];
+    await uploadDatabases(
+      testRepoName,
+      getTestConfig(tmpDir),
+      testApiDetails,
+      getRecordingLogger(loggedMessages)
+    );
+    t.assert(
+      loggedMessages.find(
+        (v) =>
+          v.type === "debug" &&
+          v.message === "Successfully uploaded database for javascript"
+      ) !== undefined
+    );
+  });
+});

src/database-upload.ts

@@ -0,0 +1,77 @@
+import * as fs from "fs";
+
+import * as actionsUtil from "./actions-util";
+import { getApiClient, GitHubApiDetails } from "./api-client";
+import { getCodeQL } from "./codeql";
+import { Config } from "./config-utils";
+import { Logger } from "./logging";
+import { RepositoryNwo } from "./repository";
+import * as util from "./util";
+
+export async function uploadDatabases(
+  repositoryNwo: RepositoryNwo,
+  config: Config,
+  apiDetails: GitHubApiDetails,
+  logger: Logger
+): Promise<void> {
+  if (actionsUtil.getRequiredInput("upload-database") !== "true") {
+    logger.debug("Database upload disabled in workflow. Skipping upload.");
+    return;
+  }
+
+  // Do nothing when not running against github.com
+  if (config.gitHubVersion.type !== util.GitHubVariant.DOTCOM) {
+    logger.debug("Not running against github.com. Skipping upload.");
+    return;
+  }
+
+  if (!(await actionsUtil.isAnalyzingDefaultBranch())) {
+    // We only want to upload a database if we are analyzing the default branch.
+    logger.debug("Not analyzing default branch. Skipping upload.");
+    return;
+  }
+
+  const client = getApiClient(apiDetails);
+  try {
+    await client.request("GET /repos/:owner/:repo/code-scanning/databases", {
+      owner: repositoryNwo.owner,
+      repo: repositoryNwo.repo,
+    });
+  } catch (e) {
+    if (util.isHTTPError(e) && e.status === 404) {
+      logger.debug(
+        "Repository is not opted in to database uploads. Skipping upload."
+      );
+    } else {
+      console.log(e);
+      logger.info(`Skipping database upload due to unknown error: ${e}`);
+    }
+    return;
+  }
+
+  const codeql = getCodeQL(config.codeQLCmd);
+  for (const language of config.languages) {
+    // Bundle the database up into a single zip file
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    const databaseBundlePath = `${databasePath}.zip`;
+    await codeql.databaseBundle(databasePath, databaseBundlePath);
+
+    // Upload the database bundle
+    const payload = fs.readFileSync(databaseBundlePath);
+    try {
+      await client.request(
+        `PUT /repos/:owner/:repo/code-scanning/databases/${language}`,
+        {
+          owner: repositoryNwo.owner,
+          repo: repositoryNwo.repo,
+          data: payload,
+        }
+      );
+      logger.debug(`Successfully uploaded database for ${language}`);
+    } catch (e) {
+      console.log(e);
+      // Log a warning but don't fail the workflow
+      logger.warning(`Failed to upload database for ${language}: ${e}`);
+    }
+  }
+}

src/util.ts

@@ -478,3 +478,16 @@ export function getRequiredEnvParam(paramName: string): string {
   }
   return value;
 }
+
+export class HTTPError extends Error {
+  public status: number;
+
+  constructor(message: string, status: number) {
+    super(message);
+    this.status = status;
+  }
+}
+
+export function isHTTPError(arg: any): arg is HTTPError {
+  return arg?.status !== undefined && Number.isInteger(arg.status);
+}