Merge pull request #1817 from github/nickrolfe/evaluator-intra-layer-parallelism-ff

Use feature flag to enable evaluator intra-layer parallelism

CHANGELOG.md

@@ -4,7 +4,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 
 ## [UNRELEASED]
 
-No user facing changes.
+- We are rolling out a feature in August 2023 that will improve multi-threaded performance on larger runners. [#1817](https://github.com/github/codeql-action/pull/1817)
 
 ## 2.21.2 - 28 Jul 2023
 

lib/analyze.js

@@ -285,7 +285,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
             fs.writeFileSync(querySuitePath, querySuiteContents);
             logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
         }
-        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun);
+        await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun, features);
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
         return querySuitePath;
     }
@@ -299,7 +299,7 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         const querySuitePath = `${databasePath}-queries-${type}.qls`;
         fs.writeFileSync(querySuitePath, yaml.dump(querySuite));
         logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
-        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun);
+        await codeql.databaseRunQueries(databasePath, undefined, querySuitePath, queryFlags, optimizeForLastQueryRun, features);
         return querySuitePath;
     }
 }

lib/codeql.js

@@ -419,7 +419,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
                 throw new Error(`Unexpected output from codeql resolve build-environment: ${e} in\n${output}`);
             }
         },
-        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun) {
+        async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun, features) {
             const codeqlArgs = [
                 "database",
                 "run-queries",
@@ -439,6 +439,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             if (querySuitePath) {
                 codeqlArgs.push(querySuitePath);
             }
+            if (await features.getValue(feature_flags_1.Feature.EvaluatorIntraLayerParallelismEnabled, this)) {
+                codeqlArgs.push("--intra-layer-parallelism");
+            }
             await runTool(cmd, codeqlArgs);
         },
         async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, verbosityFlag, automationDetailsId, config, features, logger) {

lib/feature-flags.js

@@ -50,6 +50,7 @@ var Feature;
     Feature["CliConfigFileEnabled"] = "cli_config_file_enabled";
     Feature["DisableKotlinAnalysisEnabled"] = "disable_kotlin_analysis_enabled";
     Feature["DisablePythonDependencyInstallationEnabled"] = "disable_python_dependency_installation_enabled";
+    Feature["EvaluatorIntraLayerParallelismEnabled"] = "evaluator_intra_layer_parallelism_enabled";
     Feature["ExportDiagnosticsEnabled"] = "export_diagnostics_enabled";
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     Feature["NewAnalysisSummaryEnabled"] = "new_analysis_summary_enabled";
@@ -68,6 +69,11 @@ exports.featureConfig = {
         minimumVersion: "2.11.6",
         defaultValue: true,
     },
+    [Feature.EvaluatorIntraLayerParallelismEnabled]: {
+        envVar: "CODEQL_EVALUATOR_INTRA_LAYER_PARALLELISM",
+        minimumVersion: "2.14.0",
+        defaultValue: false,
+    },
     [Feature.ExportDiagnosticsEnabled]: {
         envVar: "CODEQL_ACTION_EXPORT_DIAGNOSTICS",
         minimumVersion: "2.12.4",

src/analyze.ts

@@ -487,6 +487,7 @@ export async function runQueries(
       querySuitePath,
       queryFlags,
       optimizeForLastQueryRun,
+      features,
     );
 
     logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
@@ -521,6 +522,7 @@ export async function runQueries(
       querySuitePath,
       queryFlags,
       optimizeForLastQueryRun,
+      features,
     );
 
     return querySuitePath;

src/codeql.ts

@@ -160,6 +160,7 @@ export interface CodeQL {
     querySuitePath: string | undefined,
     flags: string[],
     optimizeForLastQueryRun: boolean,
+    features: FeatureEnablement,
   ): Promise<void>;
   /**
    * Run 'codeql database interpret-results'.
@@ -756,6 +757,7 @@ export async function getCodeQLForCmd(
       querySuitePath: string | undefined,
       flags: string[],
       optimizeForLastQueryRun: boolean,
+      features: FeatureEnablement,
     ): Promise<void> {
       const codeqlArgs = [
         "database",
@@ -778,6 +780,14 @@ export async function getCodeQLForCmd(
       if (querySuitePath) {
         codeqlArgs.push(querySuitePath);
       }
+      if (
+        await features.getValue(
+          Feature.EvaluatorIntraLayerParallelismEnabled,
+          this,
+        )
+      ) {
+        codeqlArgs.push("--intra-layer-parallelism");
+      }
       await runTool(cmd, codeqlArgs);
     },
     async databaseInterpretResults(

src/feature-flags.ts

@@ -46,6 +46,7 @@ export enum Feature {
   CliConfigFileEnabled = "cli_config_file_enabled",
   DisableKotlinAnalysisEnabled = "disable_kotlin_analysis_enabled",
   DisablePythonDependencyInstallationEnabled = "disable_python_dependency_installation_enabled",
+  EvaluatorIntraLayerParallelismEnabled = "evaluator_intra_layer_parallelism_enabled",
   ExportDiagnosticsEnabled = "export_diagnostics_enabled",
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
   NewAnalysisSummaryEnabled = "new_analysis_summary_enabled",
@@ -68,6 +69,11 @@ export const featureConfig: Record<
     minimumVersion: "2.11.6",
     defaultValue: true,
   },
+  [Feature.EvaluatorIntraLayerParallelismEnabled]: {
+    envVar: "CODEQL_EVALUATOR_INTRA_LAYER_PARALLELISM",
+    minimumVersion: "2.14.0",
+    defaultValue: false,
+  },
   [Feature.ExportDiagnosticsEnabled]: {
     envVar: "CODEQL_ACTION_EXPORT_DIAGNOSTICS",
     minimumVersion: "2.12.4",