Merge remote-tracking branch 'origin/main' into aibaars/python-setup-no-pycache

lib/analyze-action.js

@@ -26,9 +26,11 @@ const artifact = __importStar(require("@actions/artifact"));
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
+const api_client_1 = require("./api-client");
 const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const database_upload_1 = require("./database-upload");
+const feature_flags_1 = require("./feature-flags");
 const logging_1 = require("./logging");
 const repository_1 = require("./repository");
 const upload_lib = __importStar(require("./upload-lib"));
@@ -76,7 +78,9 @@ async function run() {
         const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
         const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
+        const featureFlags = new feature_flags_1.GitHubFeatureFlags(gitHubVersion, apiDetails, repositoryNwo, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, featureFlags);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
             if (config.debugMode) {

lib/analyze.js

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.runCleanup = exports.runFinalize = exports.runQueries = exports.CodeQLAnalysisError = void 0;
+exports.runCleanup = exports.runFinalize = exports.runQueries = exports.createdDBForScannedLanguages = exports.CodeQLAnalysisError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -68,11 +68,10 @@ async function setupPythonExtractor(logger) {
     logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
     process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
-async function createdDBForScannedLanguages(config, logger) {
+async function createdDBForScannedLanguages(codeql, config, logger, featureFlags) {
     // Insert the LGTM_INDEX_X env vars at this point so they are set when
     // we extract any scanned languages.
     analysisPaths.includeAndExcludeAnalysisPaths(config);
-    const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     for (const language of config.languages) {
         if ((0, languages_1.isScannedLanguage)(language) &&
             !dbIsFinalized(config, language, logger)) {
@@ -80,11 +79,12 @@ async function createdDBForScannedLanguages(config, logger) {
             if (language === languages_1.Language.python) {
                 await setupPythonExtractor(logger);
             }
-            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language);
+            await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language, featureFlags);
             logger.endGroup();
         }
     }
 }
+exports.createdDBForScannedLanguages = createdDBForScannedLanguages;
 function dbIsFinalized(config, language, logger) {
     const dbPath = util.getCodeQLDatabasePath(config, language);
     try {
@@ -96,9 +96,9 @@ function dbIsFinalized(config, language, logger) {
         return false;
     }
 }
-async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger) {
-    await createdDBForScannedLanguages(config, logger);
+async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+    await createdDBForScannedLanguages(codeql, config, logger, featureFlags);
     for (const language of config.languages) {
         if (dbIsFinalized(config, language, logger)) {
             logger.info(`There is already a finalized database for ${language} at the location where the CodeQL Action places databases, so we did not create one.`);
@@ -238,7 +238,7 @@ exports.runQueries = runQueries;
 function createQuerySuiteContents(queries) {
     return queries.map((q) => `- query: ${q}`).join("\n");
 }
-async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
+async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, featureFlags) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     if (await util.codeQlVersionAbove(codeql, codeql_1.CODEQL_VERSION_NEW_TRACING)) {
         // Delete variables as specified by the end-tracing script
@@ -257,7 +257,7 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
         }
     }
     await fs.promises.mkdir(outputDir, { recursive: true });
-    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+    await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, featureFlags);
 }
 exports.runFinalize = runFinalize;
 async function runCleanup(config, cleanupLevel, logger) {

lib/analyze.test.js

@@ -29,7 +29,9 @@ const yaml = __importStar(require("js-yaml"));
 const sinon = __importStar(require("sinon"));
 const analyze_1 = require("./analyze");
 const codeql_1 = require("./codeql");
+const codeql_test_1 = require("./codeql.test");
 const count = __importStar(require("./count-loc"));
+const feature_flags_1 = require("./feature-flags");
 const languages_1 = require("./languages");
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
@@ -210,4 +212,72 @@ const util = __importStar(require("./util"));
         }
     }
 });
+const stubConfig = {
+    languages: [languages_1.Language.cpp, languages_1.Language.go],
+    queries: {},
+    pathsIgnore: [],
+    paths: [],
+    originalUserInput: {},
+    tempDir: "",
+    toolCacheDir: "",
+    codeQLCmd: "",
+    gitHubVersion: {
+        type: util.GitHubVariant.DOTCOM,
+    },
+    dbLocation: "",
+    packs: {},
+    debugMode: false,
+    debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
+    debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
+    injectedMlQueries: false,
+};
+for (const options of [
+    {
+        name: "Lua feature flag enabled, but old CLI",
+        version: "2.9.0",
+        featureFlags: [feature_flags_1.FeatureFlag.LuaTracerConfigEnabled],
+        yesFlagSet: false,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag disabled, with old CLI",
+        version: "2.9.0",
+        featureFlags: [],
+        yesFlagSet: false,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag enabled, with new CLI",
+        version: "2.10.0",
+        featureFlags: [feature_flags_1.FeatureFlag.LuaTracerConfigEnabled],
+        yesFlagSet: true,
+        noFlagSet: false,
+    },
+    {
+        name: "Lua feature flag disabled, with new CLI",
+        version: "2.10.0",
+        featureFlags: [],
+        yesFlagSet: false,
+        noFlagSet: true,
+    },
+]) {
+    (0, ava_1.default)(`createdDBForScannedLanguages() ${options.name}`, async (t) => {
+        const runnerConstructorStub = (0, codeql_test_1.stubToolRunnerConstructor)();
+        const codeqlObject = await (0, codeql_1.getCodeQLForTesting)("codeql/for-testing");
+        sinon.stub(codeqlObject, "getVersion").resolves(options.version);
+        const promise = (0, analyze_1.createdDBForScannedLanguages)(codeqlObject, stubConfig, (0, logging_1.getRunnerLogger)(true), (0, feature_flags_1.createFeatureFlags)(options.featureFlags));
+        // call listener on `codeql resolve extractor`
+        const mockToolRunner = runnerConstructorStub.getCall(0);
+        mockToolRunner.args[2].listeners.stdout('"/path/to/extractor"');
+        await promise;
+        if (options.yesFlagSet)
+            t.true(runnerConstructorStub.secondCall.args[1].includes("--internal-use-lua-tracing"), "--internal-use-lua-tracing should be present, but it is absent");
+        else
+            t.false(runnerConstructorStub.secondCall.args[1].includes("--internal-use-lua-tracing"), "--internal-use-lua-tracing should be absent, but it is present");
+        if (options.noFlagSet)
+            t.true(runnerConstructorStub.secondCall.args[1].includes("--no-internal-use-lua-tracing"), "--no-internal-use-lua-tracing should be present, but it is absent");
+        else
+            t.false(runnerConstructorStub.secondCall.args[1].includes("--no-internal-use-lua-tracing"), "--no-internal-use-lua-tracing should be absent, but it is present");
+    });
+}
 //# sourceMappingURL=analyze.test.js.map

lib/codeql.js

@@ -384,8 +384,8 @@ exports.getCachedCodeQL = getCachedCodeQL;
  * a non-existent placeholder codeql command, so tests that use this function
  * should also stub the toolrunner.ToolRunner constructor.
  */
-async function getCodeQLForTesting() {
-    return getCodeQLForCmd("codeql-for-testing", false);
+async function getCodeQLForTesting(cmd = "codeql-for-testing") {
+    return getCodeQLForCmd(cmd, false);
 }
 exports.getCodeQLForTesting = getCodeQLForTesting;
 /**
@@ -510,7 +510,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             ].join(" ");
             await runTool(autobuildCmd);
         },
-        async extractScannedLanguage(databasePath, language) {
+        async extractScannedLanguage(databasePath, language, featureFlags) {
             // Get extractor location
             let extractorPath = "";
             await new toolrunner.ToolRunner(cmd, [
@@ -533,10 +533,20 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             // Set trace command
             const ext = process.platform === "win32" ? ".cmd" : ".sh";
             const traceCommand = path.resolve(JSON.parse(extractorPath), "tools", `autobuild${ext}`);
+            const extraArgs = [];
+            if (await util.codeQlVersionAbove(this, CODEQL_VERSION_LUA_TRACER_CONFIG)) {
+                if (await featureFlags.getValue(feature_flags_1.FeatureFlag.LuaTracerConfigEnabled)) {
+                    extraArgs.push("--internal-use-lua-tracing");
+                }
+                else {
+                    extraArgs.push("--no-internal-use-lua-tracing");
+                }
+            }
             // Run trace command
             await (0, toolrunner_error_catcher_1.toolrunnerErrorCatcher)(cmd, [
                 "database",
                 "trace-command",
+                ...extraArgs,
                 ...getExtraOptionsFromEnv(["database", "trace-command"]),
                 databasePath,
                 "--",

lib/codeql.test.js

@@ -22,6 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.stubToolRunnerConstructor = void 0;
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
 const toolcache = __importStar(require("@actions/tool-cache"));
@@ -291,4 +292,5 @@ function stubToolRunnerConstructor() {
     runnerConstructorStub.returns(runnerObjectStub);
     return runnerConstructorStub;
 }
+exports.stubToolRunnerConstructor = stubToolRunnerConstructor;
 //# sourceMappingURL=codeql.test.js.map

lib/defaults.json

@@ -1,3 +1,3 @@
 {
-    "bundleVersion": "codeql-bundle-20220615"
+    "bundleVersion": "codeql-bundle-20220623"
 }

lib/runner.js

@@ -297,7 +297,7 @@ program
         }
         const threads = (0, util_1.getThreadsFlag)(cmd.threads || initEnv["CODEQL_THREADS"], logger);
         const memory = (0, util_1.getMemoryFlag)(cmd.ram || initEnv["CODEQL_RAM"]);
-        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, (0, feature_flags_1.createFeatureFlags)([]));
         await (0, analyze_1.runQueries)(outputDir, memory, (0, util_1.getAddSnippetsFlag)(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {
             logger.info("Not uploading results");