Merge branch 'main' into henrymercer/csharp-buildless-rollback-mechanism

src/actions-util.ts

@@ -476,3 +476,7 @@ export const getFileType = async (filePath: string): Promise<string> => {
     throw e;
   }
 };
+
+export function isSelfHostedRunner() {
+  return process.env.RUNNER_ENVIRONMENT === "self-hosted";
+}

src/codeql.ts

@@ -612,12 +612,19 @@ export async function getCodeQLForCmd(
         extraArgs.push("--no-sublanguage-file-coverage");
       }
 
+      const overwriteFlag = isSupportedToolsFeature(
+        await this.getVersion(),
+        ToolsFeature.ForceOverwrite,
+      )
+        ? "--force-overwrite"
+        : "--overwrite";
+
       await runTool(
         cmd,
         [
           "database",
           "init",
-          "--overwrite",
+          overwriteFlag,
           "--db-cluster",
           config.dbLocation,
           `--source-root=${sourceRoot}`,
@@ -681,6 +688,8 @@ export async function getCodeQLForCmd(
           "database",
           "trace-command",
           "--use-build-mode",
+          "--working-dir",
+          process.cwd(),
           ...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
           ...getExtractionVerbosityArguments(config.debugMode),
           ...getExtraOptionsFromEnv(["database", "trace-command"]),

src/database-upload.ts

@@ -20,6 +20,11 @@ export async function uploadDatabases(
     return;
   }
 
+  if (util.isInTestMode()) {
+    logger.debug("In test mode. Skipping database upload.");
+    return;
+  }
+
   // Do nothing when not running against github.com
   if (
     config.gitHubVersion.type !== util.GitHubVariant.DOTCOM &&

src/defaults.json

@@ -1,6 +1,6 @@
 {
-  "bundleVersion": "codeql-bundle-v2.17.4",
-  "cliVersion": "2.17.4",
-  "priorBundleVersion": "codeql-bundle-v2.17.3",
-  "priorCliVersion": "2.17.3"
+  "bundleVersion": "codeql-bundle-v2.17.5",
+  "cliVersion": "2.17.5",
+  "priorBundleVersion": "codeql-bundle-v2.17.4",
+  "priorCliVersion": "2.17.4"
 }

src/feature-flags.ts

@@ -45,7 +45,7 @@ export interface FeatureEnablement {
  * Legacy features should end with `_enabled`.
  */
 export enum Feature {
-  AutobuildDirectTracing = "autobuild_direct_tracing",
+  AutobuildDirectTracing = "autobuild_direct_tracing_v2",
   CleanupTrapCaches = "cleanup_trap_caches",
   CppDependencyInstallation = "cpp_dependency_installation_enabled",
   CppTrapCachingEnabled = "cpp_trap_caching_enabled",

src/init-action-post-helper.ts

@@ -1,3 +1,5 @@
+import * as fs from "fs";
+
 import * as core from "@actions/core";
 import * as github from "@actions/github";
 
@@ -217,6 +219,28 @@ export async function run(
     await printDebugLogs(config);
   }
 
+  if (actionsUtil.isSelfHostedRunner()) {
+    try {
+      fs.rmSync(config.dbLocation, {
+        recursive: true,
+        force: true,
+        maxRetries: 3,
+      });
+      logger.info(
+        `Cleaned up database cluster directory ${config.dbLocation}.`,
+      );
+    } catch (e) {
+      logger.warning(
+        `Failed to clean up database cluster directory ${config.dbLocation}. Details: ${e}`,
+      );
+    }
+  } else {
+    logger.debug(
+      "Skipping cleanup of database cluster directory since we are running on a GitHub-hosted " +
+        "runner which will be automatically cleaned up.",
+    );
+  }
+
   return uploadFailedSarifResult;
 }
 

src/init-action.ts

@@ -26,6 +26,7 @@ import { EnvVar } from "./environment";
 import { Feature, Features } from "./feature-flags";
 import {
   checkInstallPython311,
+  cleanupDatabaseClusterDirectory,
   initCodeQL,
   initConfig,
   isSipEnabled,
@@ -321,6 +322,8 @@ async function run() {
   }
 
   try {
+    cleanupDatabaseClusterDirectory(config, logger);
+
     // Forward Go flags
     const goFlags = process.env["GOFLAGS"];
     if (goFlags) {

src/init.test.ts

@@ -1,9 +1,21 @@
+import * as fs from "fs";
+import path from "path";
+
 import test from "ava";
 
 import { Config } from "./config-utils";
-import { printPathFiltersWarning } from "./init";
+import {
+  cleanupDatabaseClusterDirectory,
+  printPathFiltersWarning,
+} from "./init";
 import { Language } from "./languages";
-import { LoggedMessage, getRecordingLogger, setupTests } from "./testing-utils";
+import {
+  LoggedMessage,
+  createTestConfig,
+  getRecordingLogger,
+  setupTests,
+} from "./testing-utils";
+import { ConfigurationError, withTmpDir } from "./util";
 
 setupTests(test);
 
@@ -30,3 +42,96 @@ test("printPathFiltersWarning does not trigger when 'paths' and 'paths-ignore' a
   );
   t.is(messages.length, 0);
 });
+
+test("cleanupDatabaseClusterDirectory cleans up where possible", async (t) => {
+  await withTmpDir(async (tmpDir: string) => {
+    const dbLocation = path.resolve(tmpDir, "dbs");
+    fs.mkdirSync(dbLocation, { recursive: true });
+
+    const fileToCleanUp = path.resolve(dbLocation, "something-to-cleanup.txt");
+    fs.writeFileSync(fileToCleanUp, "");
+
+    const messages: LoggedMessage[] = [];
+    cleanupDatabaseClusterDirectory(
+      createTestConfig({ dbLocation }),
+      getRecordingLogger(messages),
+    );
+
+    t.is(messages.length, 2);
+    t.is(messages[0].type, "warning");
+    t.is(
+      messages[0].message,
+      `The database cluster directory ${dbLocation} must be empty. Attempting to clean it up.`,
+    );
+    t.is(messages[1].type, "info");
+    t.is(
+      messages[1].message,
+      `Cleaned up database cluster directory ${dbLocation}.`,
+    );
+
+    t.false(fs.existsSync(fileToCleanUp));
+  });
+});
+
+for (const { runnerEnv, ErrorConstructor, message } of [
+  {
+    runnerEnv: "self-hosted",
+    ErrorConstructor: ConfigurationError,
+    message: (dbLocation) =>
+      "The CodeQL Action requires an empty database cluster directory. By default, this is located " +
+      `at ${dbLocation}. You can customize it using the 'db-location' input to the init Action. An ` +
+      "attempt was made to clean up the directory, but this failed. This can happen if another " +
+      "process is using the directory or the directory is owned by a different user. Please clean " +
+      "up the directory manually and rerun the job.",
+  },
+  {
+    runnerEnv: "github-hosted",
+    ErrorConstructor: Error,
+    message: (dbLocation) =>
+      "The CodeQL Action requires an empty database cluster directory. By default, this is located " +
+      `at ${dbLocation}. You can customize it using the 'db-location' input to the init Action. An ` +
+      "attempt was made to clean up the directory, but this failed. This shouldn't typically " +
+      "happen on hosted runners. If you are using an advanced setup, please check your workflow, " +
+      "otherwise we recommend rerunning the job.",
+  },
+]) {
+  test(`cleanupDatabaseClusterDirectory throws a ${ErrorConstructor.name} when cleanup fails on ${runnerEnv} runner`, async (t) => {
+    await withTmpDir(async (tmpDir: string) => {
+      process.env["RUNNER_ENVIRONMENT"] = runnerEnv;
+
+      const dbLocation = path.resolve(tmpDir, "dbs");
+      fs.mkdirSync(dbLocation, { recursive: true });
+
+      const fileToCleanUp = path.resolve(
+        dbLocation,
+        "something-to-cleanup.txt",
+      );
+      fs.writeFileSync(fileToCleanUp, "");
+
+      const rmSyncError = `Failed to clean up file ${fileToCleanUp}`;
+
+      const messages: LoggedMessage[] = [];
+      t.throws(
+        () =>
+          cleanupDatabaseClusterDirectory(
+            createTestConfig({ dbLocation }),
+            getRecordingLogger(messages),
+            () => {
+              throw new Error(rmSyncError);
+            },
+          ),
+        {
+          instanceOf: ErrorConstructor,
+          message: `${message(dbLocation)} Details: ${rmSyncError}`,
+        },
+      );
+
+      t.is(messages.length, 1);
+      t.is(messages[0].type, "warning");
+      t.is(
+        messages[0].message,
+        `The database cluster directory ${dbLocation} must be empty. Attempting to clean it up.`,
+      );
+    });
+  });
+}

src/init.ts

@@ -5,6 +5,7 @@ import * as exec from "@actions/exec/lib/exec";
 import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as safeWhich from "@chrisgavin/safe-which";
 
+import { getOptionalInput, isSelfHostedRunner } from "./actions-util";
 import { GitHubApiCombinedDetails, GitHubApiDetails } from "./api-client";
 import { CodeQL, setupCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
@@ -171,3 +172,57 @@ export async function isSipEnabled(logger): Promise<boolean | undefined> {
     return undefined;
   }
 }
+
+export function cleanupDatabaseClusterDirectory(
+  config: configUtils.Config,
+  logger: Logger,
+  // We can't stub the fs module in tests, so we allow the caller to override the rmSync function
+  // for testing.
+  rmSync = fs.rmSync,
+): void {
+  if (
+    fs.existsSync(config.dbLocation) &&
+    (fs.statSync(config.dbLocation).isFile() ||
+      fs.readdirSync(config.dbLocation).length)
+  ) {
+    logger.warning(
+      `The database cluster directory ${config.dbLocation} must be empty. Attempting to clean it up.`,
+    );
+    try {
+      rmSync(config.dbLocation, {
+        force: true,
+        maxRetries: 3,
+        recursive: true,
+      });
+
+      logger.info(
+        `Cleaned up database cluster directory ${config.dbLocation}.`,
+      );
+    } catch (e) {
+      const blurb = `The CodeQL Action requires an empty database cluster directory. ${
+        getOptionalInput("db-location")
+          ? `This is currently configured to be ${config.dbLocation}. `
+          : `By default, this is located at ${config.dbLocation}. ` +
+            "You can customize it using the 'db-location' input to the init Action. "
+      }An attempt was made to clean up the directory, but this failed.`;
+
+      // Hosted runners are automatically cleaned up, so this error should not occur for hosted runners.
+      if (isSelfHostedRunner()) {
+        throw new util.ConfigurationError(
+          `${blurb} This can happen if another process is using the directory or the directory is owned by a different user. ` +
+            `Please clean up the directory manually and rerun the job. Details: ${
+              util.wrapError(e).message
+            }`,
+        );
+      } else {
+        throw new Error(
+          `${blurb} This shouldn't typically happen on hosted runners. ` +
+            "If you are using an advanced setup, please check your workflow, otherwise we " +
+            `recommend rerunning the job. Details: ${
+              util.wrapError(e).message
+            }`,
+        );
+      }
+    }
+  }
+}

src/tools-features.ts

@@ -8,6 +8,7 @@ export enum ToolsFeature {
   SetsCodeqlRunnerEnvVar = "setsCodeqlRunnerEnvVar",
   TraceCommandUseBuildMode = "traceCommandUseBuildMode",
   SarifMergeRunsFromEqualCategory = "sarifMergeRunsFromEqualCategory",
+  ForceOverwrite = "forceOverwrite",
 }
 
 /**