Merge pull request #1676 from github/rasmuswl/python-disable-dependency-installation

Feature flag to disable python dependency installation

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## [UNRELEASED]
 
+- We are rolling out a feature in May 2023 that will disable Python dependency installation for new users of the CodeQL Action. This improves the speed of analysis while having only a very minor impact on results. [#1676](https://github.com/github/codeql-action/pull/1676)
 - Remove the requirement for `on.push` and `on.pull_request` to trigger on the same branches. [#1675](https://github.com/github/codeql-action/pull/1675)
 
 ## 2.3.3 - 04 May 2023

lib/analyze-action.js

@@ -163,7 +163,7 @@ async function run() {
         const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
         const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
         await runAutobuildIfLegacyGoWorkflow(config, logger);
-        dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
+        dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, features);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger, features);
         }

lib/analyze.js

@@ -36,6 +36,7 @@ const yaml = __importStar(require("js-yaml"));
 const analysisPaths = __importStar(require("./analysis-paths"));
 const codeql_1 = require("./codeql");
 const configUtils = __importStar(require("./config-utils"));
+const feature_flags_1 = require("./feature-flags");
 const languages_1 = require("./languages");
 const tracer_config_1 = require("./tracer-config");
 const util = __importStar(require("./util"));
@@ -47,12 +48,17 @@ class CodeQLAnalysisError extends Error {
     }
 }
 exports.CodeQLAnalysisError = CodeQLAnalysisError;
-async function setupPythonExtractor(logger) {
+async function setupPythonExtractor(logger, features, codeql) {
     const codeqlPython = process.env["CODEQL_PYTHON"];
     if (codeqlPython === undefined || codeqlPython.length === 0) {
         // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
         return;
     }
+    if (await features.getValue(feature_flags_1.Feature.DisablePythonDependencyInstallation, codeql)) {
+        logger.warning("We recommend that you remove the CODEQL_PYTHON environment variable from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
+            "\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'.");
+        return;
+    }
     const scriptsFolder = path.resolve(__dirname, "../python-setup");
     let output = "";
     const options = {
@@ -70,7 +76,7 @@ async function setupPythonExtractor(logger) {
     logger.info(`Setting LGTM_PYTHON_SETUP_VERSION=${output}`);
     process.env["LGTM_PYTHON_SETUP_VERSION"] = output;
 }
-async function createdDBForScannedLanguages(codeql, config, logger) {
+async function createdDBForScannedLanguages(codeql, config, logger, features) {
     // Insert the LGTM_INDEX_X env vars at this point so they are set when
     // we extract any scanned languages.
     analysisPaths.includeAndExcludeAnalysisPaths(config);
@@ -79,7 +85,7 @@ async function createdDBForScannedLanguages(codeql, config, logger) {
             !dbIsFinalized(config, language, logger)) {
             logger.startGroup(`Extracting ${language}`);
             if (language === languages_1.Language.python) {
-                await setupPythonExtractor(logger);
+                await setupPythonExtractor(logger, features, codeql);
             }
             await codeql.extractScannedLanguage(config, language);
             logger.endGroup();
@@ -99,10 +105,10 @@ function dbIsFinalized(config, language, logger) {
     }
 }
 exports.dbIsFinalized = dbIsFinalized;
-async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger) {
+async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, features) {
     const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
     const extractionStart = perf_hooks_1.performance.now();
-    await createdDBForScannedLanguages(codeql, config, logger);
+    await createdDBForScannedLanguages(codeql, config, logger, features);
     const extractionTime = perf_hooks_1.performance.now() - extractionStart;
     const trapImportStart = perf_hooks_1.performance.now();
     for (const language of config.languages) {
@@ -271,7 +277,7 @@ function createQuerySuiteContents(queries, queryFilters) {
     return yaml.dump(queries.map((q) => ({ query: q })).concat(queryFilters));
 }
 exports.createQuerySuiteContents = createQuerySuiteContents;
-async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
+async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, features) {
     try {
         await (0, del_1.default)(outputDir, { force: true });
     }
@@ -281,7 +287,7 @@ async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger) {
         }
     }
     await fs.promises.mkdir(outputDir, { recursive: true });
-    const timings = await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger);
+    const timings = await finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger, features);
     // WARNING: This does not _really_ end tracing, as the tracer will restore its
     // critical environment variables and it'll still be active for all processes
     // launched from this build step.

lib/feature-flags.js

@@ -40,6 +40,7 @@ var Feature;
     Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
+    Feature["DisablePythonDependencyInstallation"] = "disable_python_dependency_installation";
 })(Feature = exports.Feature || (exports.Feature = {}));
 exports.featureConfig = {
     [Feature.DisableKotlinAnalysisEnabled]: {
@@ -72,6 +73,16 @@ exports.featureConfig = {
         minimumVersion: "2.11.3",
         defaultValue: true,
     },
+    [Feature.DisablePythonDependencyInstallation]: {
+        envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
+        // Although the python extractor only started supporting not extracting installed
+        // dependencies in 2.13.1, the init-action can still benefit from not installing
+        // dependencies no matter what codeql version we are using, so therefore the
+        // minimumVersion is set to 'undefined'. This means that with an old CodeQL version,
+        // packages available with current python3 installation might get extracted.
+        minimumVersion: undefined,
+        defaultValue: false,
+    },
 };
 exports.FEATURE_FLAGS_FILE_NAME = "cached-feature-flags.json";
 /**

lib/init-action.js

@@ -136,12 +136,17 @@ async function run() {
         (0, actions_util_1.getOptionalInput)("debug") === "true" || core.isDebug(), (0, actions_util_1.getOptionalInput)("debug-artifact-name") || util_1.DEFAULT_DEBUG_ARTIFACT_NAME, (0, actions_util_1.getOptionalInput)("debug-database-name") || util_1.DEFAULT_DEBUG_DATABASE_NAME, repositoryNwo, (0, actions_util_1.getTemporaryDirectory)(), codeql, (0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), gitHubVersion, apiDetails, features, logger);
         if (config.languages.includes(languages_1.Language.python) &&
             (0, actions_util_1.getRequiredInput)("setup-python-dependencies") === "true") {
-            try {
-                await (0, init_1.installPythonDeps)(codeql, logger);
+            if (await features.getValue(feature_flags_1.Feature.DisablePythonDependencyInstallation, codeql)) {
+                logger.info("Skipping python dependency installation");
             }
-            catch (unwrappedError) {
-                const error = (0, util_1.wrapError)(unwrappedError);
-                logger.warning(`${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`);
+            else {
+                try {
+                    await (0, init_1.installPythonDeps)(codeql, logger);
+                }
+                catch (unwrappedError) {
+                    const error = (0, util_1.wrapError)(unwrappedError);
+                    logger.warning(`${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`);
+                }
             }
         }
     }
@@ -170,6 +175,10 @@ async function run() {
         if (await features.getValue(feature_flags_1.Feature.DisableKotlinAnalysisEnabled)) {
             core.exportVariable("CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN", "true");
         }
+        // Disable Python dependency extraction if feature flag set
+        if (await features.getValue(feature_flags_1.Feature.DisablePythonDependencyInstallation, codeql)) {
+            core.exportVariable("CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION", "true");
+        }
         const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
         const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", registriesInput, features, apiDetails, logger);
         if (tracerConfig !== undefined) {

src/analyze-action.ts

@@ -237,7 +237,8 @@ async function run() {
       threads,
       memory,
       config,
-      logger
+      logger,
+      features
     );
 
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

src/analyze.ts

@@ -10,7 +10,7 @@ import { DatabaseCreationTimings } from "./actions-util";
 import * as analysisPaths from "./analysis-paths";
 import { CodeQL, getCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
-import { FeatureEnablement } from "./feature-flags";
+import { FeatureEnablement, Feature } from "./feature-flags";
 import { isScannedLanguage, Language } from "./languages";
 import { Logger } from "./logging";
 import { endTracingForCluster } from "./tracer-config";
@@ -80,13 +80,27 @@ export interface QueriesStatusReport {
   analyze_failure_language?: string;
 }
 
-async function setupPythonExtractor(logger: Logger) {
+async function setupPythonExtractor(
+  logger: Logger,
+  features: FeatureEnablement,
+  codeql: CodeQL
+) {
   const codeqlPython = process.env["CODEQL_PYTHON"];
   if (codeqlPython === undefined || codeqlPython.length === 0) {
     // If CODEQL_PYTHON is not set, no dependencies were installed, so we don't need to do anything
     return;
   }
 
+  if (
+    await features.getValue(Feature.DisablePythonDependencyInstallation, codeql)
+  ) {
+    logger.warning(
+      "We recommend that you remove the CODEQL_PYTHON environment variable from your workflow. This environment variable was originally used to specify a Python executable that included the dependencies of your Python code, however Python analysis no longer uses these dependencies." +
+        "\nIf you used CODEQL_PYTHON to force the version of Python to analyze as, please use CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION instead, such as 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=2.7' or 'CODEQL_EXTRACTOR_PYTHON_ANALYSIS_VERSION=3.11'."
+    );
+    return;
+  }
+
   const scriptsFolder = path.resolve(__dirname, "../python-setup");
 
   let output = "";
@@ -119,7 +133,8 @@ async function setupPythonExtractor(logger: Logger) {
 export async function createdDBForScannedLanguages(
   codeql: CodeQL,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  features: FeatureEnablement
 ) {
   // Insert the LGTM_INDEX_X env vars at this point so they are set when
   // we extract any scanned languages.
@@ -133,7 +148,7 @@ export async function createdDBForScannedLanguages(
       logger.startGroup(`Extracting ${language}`);
 
       if (language === Language.python) {
-        await setupPythonExtractor(logger);
+        await setupPythonExtractor(logger, features, codeql);
       }
 
       await codeql.extractScannedLanguage(config, language);
@@ -165,12 +180,13 @@ async function finalizeDatabaseCreation(
   config: configUtils.Config,
   threadsFlag: string,
   memoryFlag: string,
-  logger: Logger
+  logger: Logger,
+  features: FeatureEnablement
 ): Promise<DatabaseCreationTimings> {
   const codeql = await getCodeQL(config.codeQLCmd);
 
   const extractionStart = performance.now();
-  await createdDBForScannedLanguages(codeql, config, logger);
+  await createdDBForScannedLanguages(codeql, config, logger, features);
   const extractionTime = performance.now() - extractionStart;
 
   const trapImportStart = performance.now();
@@ -474,7 +490,8 @@ export async function runFinalize(
   threadsFlag: string,
   memoryFlag: string,
   config: configUtils.Config,
-  logger: Logger
+  logger: Logger,
+  features: FeatureEnablement
 ): Promise<DatabaseCreationTimings> {
   try {
     await del(outputDir, { force: true });
@@ -489,7 +506,8 @@ export async function runFinalize(
     config,
     threadsFlag,
     memoryFlag,
-    logger
+    logger,
+    features
   );
 
   // WARNING: This does not _really_ end tracing, as the tracer will restore its

src/feature-flags.ts

@@ -43,6 +43,7 @@ export enum Feature {
   ExportDiagnosticsEnabled = "export_diagnostics_enabled",
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
   UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
+  DisablePythonDependencyInstallation = "disable_python_dependency_installation",
 }
 
 export const featureConfig: Record<
@@ -80,6 +81,16 @@ export const featureConfig: Record<
     minimumVersion: "2.11.3",
     defaultValue: true,
   },
+  [Feature.DisablePythonDependencyInstallation]: {
+    envVar: "CODEQL_ACTION_DISABLE_PYTHON_DEPENDENCY_INSTALLATION",
+    // Although the python extractor only started supporting not extracting installed
+    // dependencies in 2.13.1, the init-action can still benefit from not installing
+    // dependencies no matter what codeql version we are using, so therefore the
+    // minimumVersion is set to 'undefined'. This means that with an old CodeQL version,
+    // packages available with current python3 installation might get extracted.
+    minimumVersion: undefined,
+    defaultValue: false,
+  },
 };
 
 /**

src/init-action.ts

@@ -277,13 +277,22 @@ async function run() {
       config.languages.includes(Language.python) &&
       getRequiredInput("setup-python-dependencies") === "true"
     ) {
-      try {
-        await installPythonDeps(codeql, logger);
-      } catch (unwrappedError) {
-        const error = wrapError(unwrappedError);
-        logger.warning(
-          `${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`
-        );
+      if (
+        await features.getValue(
+          Feature.DisablePythonDependencyInstallation,
+          codeql
+        )
+      ) {
+        logger.info("Skipping python dependency installation");
+      } else {
+        try {
+          await installPythonDeps(codeql, logger);
+        } catch (unwrappedError) {
+          const error = wrapError(unwrappedError);
+          logger.warning(
+            `${error.message} You can call this action with 'setup-python-dependencies: false' to disable this process`
+          );
+        }
       }
     }
   } catch (unwrappedError) {
@@ -331,6 +340,19 @@ async function run() {
       core.exportVariable("CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN", "true");
     }
 
+    // Disable Python dependency extraction if feature flag set
+    if (
+      await features.getValue(
+        Feature.DisablePythonDependencyInstallation,
+        codeql
+      )
+    ) {
+      core.exportVariable(
+        "CODEQL_EXTRACTOR_PYTHON_DISABLE_LIBRARY_EXTRACTION",
+        "true"
+      );
+    }
+
     const sourceRoot = path.resolve(
       getRequiredEnvParam("GITHUB_WORKSPACE"),
       getOptionalInput("source-root") || ""