Merge pull request #1760 from github/nickrolfe/scaling-memory

Respect `scaling_reserved_ram` feature flag

.github/workflows/__scaling-reserved-ram.yml

@@ -0,0 +1,98 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     (cd pr-checks; pip install ruamel.yaml && python3 sync.py)
+# to regenerate this file.
+
+name: PR Check - Scaling reserved RAM
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+  CODEQL_EXTRACTOR_JAVA_AGENT_DISABLE_KOTLIN: 'true'
+on:
+  push:
+    branches:
+    - main
+    - releases/v2
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  scaling-reserved-ram:
+    strategy:
+      matrix:
+        include:
+        - os: ubuntu-latest
+          version: stable-20220401
+        - os: macos-latest
+          version: stable-20220401
+        - os: ubuntu-latest
+          version: stable-20220615
+        - os: macos-latest
+          version: stable-20220615
+        - os: ubuntu-latest
+          version: stable-20220908
+        - os: macos-latest
+          version: stable-20220908
+        - os: ubuntu-latest
+          version: stable-20221211
+        - os: macos-latest
+          version: stable-20221211
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: ubuntu-latest
+          version: latest
+        - os: macos-latest
+          version: latest
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
+    name: Scaling reserved RAM
+    permissions:
+      contents: read
+      security-events: write
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/actions/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - name: Set environment variable for Swift enablement
+      if: >-
+        runner.os != 'Windows' && (
+            matrix.version == '20220908' ||
+            matrix.version == '20221211'
+        )
+      shell: bash
+      run: echo "CODEQL_ENABLE_EXPERIMENTAL_FEATURES_SWIFT=true" >> $GITHUB_ENV
+    - uses: ./../action/init
+      id: init
+      with:
+        db-location: ${{ runner.temp }}/customDbLocation
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+    - uses: ./../action/.github/actions/setup-swift
+      with:
+        codeql-path: ${{ steps.init.outputs.codeql-path }}
+
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+
+    - uses: ./../action/analyze
+      id: analysis
+      with:
+        upload-database: false
+    env:
+      CODEQL_ACTION_SCALING_RESERVED_RAM: true
+      CODEQL_ACTION_TEST_MODE: true

CHANGELOG.md

@@ -5,6 +5,7 @@
 - This is the last release of the Action that supports CodeQL CLI versions 2.8.5 to 2.9.3. These versions of the CodeQL CLI were deprecated on June 20, 2023 alongside GitHub Enterprise Server 3.5 and will not be supported by the next release of the CodeQL Action (2.21.0).
   - If you are using one of these versions, please update to CodeQL CLI version 2.9.4 or later. For instance, if you have specified a custom version of the CLI using the 'tools' input to the 'init' Action, you can remove this input to use the default version.
   - Alternatively, if you want to continue using a version of the CodeQL CLI between 2.8.5 and 2.9.3, you can replace 'github/codeql-action/*@v2' by 'github/codeql-action/*@v2.20.4' in your code scanning workflow to ensure you continue using this version of the CodeQL Action.
+- We are rolling out a feature in July 2023 that will slightly reduce the default amount of RAM used for query execution, in proportion to the runner's total memory. This will help to avoid out-of-memory failures on larger runners. [#1760](https://github.com/github/codeql-action/pull/1760)
 
 ## 2.20.3 - 06 Jul 2023
 

lib/analyze-action.js

@@ -158,10 +158,10 @@ async function run() {
         const apiDetails = (0, api_client_1.getApiDetails)();
         const outputDir = actionsUtil.getRequiredInput("output");
         const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
-        const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
         const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
         const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
+        const memory = await util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"], features);
         await runAutobuildIfLegacyGoWorkflow(config, logger);
         dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, features);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

lib/feature-flags.js

@@ -42,6 +42,7 @@ var Feature;
     Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     Feature["NewAnalysisSummaryEnabled"] = "new_analysis_summary_enabled";
     Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
+    Feature["ScalingReservedRam"] = "scaling_reserved_ram";
     Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
 })(Feature = exports.Feature || (exports.Feature = {}));
 exports.featureConfig = {
@@ -75,6 +76,11 @@ exports.featureConfig = {
         minimumVersion: undefined,
         defaultValue: false,
     },
+    [Feature.ScalingReservedRam]: {
+        envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
+        minimumVersion: undefined,
+        defaultValue: false,
+    },
     [Feature.UploadFailedSarifEnabled]: {
         envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
         minimumVersion: "2.11.3",

lib/init-action.js

@@ -170,7 +170,7 @@ async function run() {
         // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
         // for details.
         core.exportVariable("CODEQL_RAM", process.env["CODEQL_RAM"] ||
-            (0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram")).toString());
+            (await (0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram"), features)).toString());
         core.exportVariable("CODEQL_THREADS", (0, util_1.getThreadsFlagValue)((0, actions_util_1.getOptionalInput)("threads"), logger).toString());
         // Disable Kotlin extractor if feature flag set
         if (await features.getValue(feature_flags_1.Feature.DisableKotlinAnalysisEnabled)) {

lib/util.js

@@ -108,9 +108,18 @@ exports.withTmpDir = withTmpDir;
  * from committing too much of the available memory to CodeQL.
  * @returns number
  */
-function getSystemReservedMemoryMegaBytes() {
+async function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, features) {
     // Windows needs more memory for OS processes.
-    return 1024 * (process.platform === "win32" ? 1.5 : 1);
+    const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+    if (await features.getValue(feature_flags_1.Feature.ScalingReservedRam)) {
+        // Reserve an additional 2% of the total memory, since the amount used by
+        // the kernel for page tables scales with the size of physical memory.
+        const scaledAmount = 0.02 * totalMemoryMegaBytes;
+        return fixedAmount + scaledAmount;
+    }
+    else {
+        return fixedAmount;
+    }
 }
 /**
  * Get the value of the codeql `--ram` flag as configured by the `ram` input.
@@ -119,7 +128,7 @@ function getSystemReservedMemoryMegaBytes() {
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-function getMemoryFlagValue(userInput) {
+async function getMemoryFlagValue(userInput, features) {
     let memoryToUseMegaBytes;
     if (userInput) {
         memoryToUseMegaBytes = Number(userInput);
@@ -130,7 +139,7 @@ function getMemoryFlagValue(userInput) {
     else {
         const totalMemoryBytes = os.totalmem();
         const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
-        const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
+        const reservedMemoryMegaBytes = await getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, features);
         memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
     }
     return Math.floor(memoryToUseMegaBytes);
@@ -143,8 +152,9 @@ exports.getMemoryFlagValue = getMemoryFlagValue;
  *
  * @returns string
  */
-function getMemoryFlag(userInput) {
-    return `--ram=${getMemoryFlagValue(userInput)}`;
+async function getMemoryFlag(userInput, features) {
+    const megabytes = await getMemoryFlagValue(userInput, features);
+    return `--ram=${megabytes}`;
 }
 exports.getMemoryFlag = getMemoryFlag;
 /**

lib/util.test.js

@@ -33,6 +33,7 @@ const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
 const sinon = __importStar(require("sinon"));
 const api = __importStar(require("./api-client"));
+const feature_flags_1 = require("./feature-flags");
 const logging_1 = require("./logging");
 const testing_utils_1 = require("./testing-utils");
 const util = __importStar(require("./util"));
@@ -42,22 +43,28 @@ const util = __importStar(require("./util"));
     const toolNames = util.getToolNames(JSON.parse(input));
     t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
-(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", (t) => {
-    const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
-    const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
+(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", async (t) => {
+    const totalMem = os.totalmem() / (1024 * 1024);
+    const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+    const scaledAmount = 0.02 * totalMem;
+    const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+    const expectedMemoryValueWithScaling = Math.floor(totalMem - fixedAmount - scaledAmount);
     const tests = [
-        [undefined, `--ram=${totalMem - expectedThreshold}`],
-        ["", `--ram=${totalMem - expectedThreshold}`],
-        ["512", "--ram=512"],
+        [undefined, false, `--ram=${expectedMemoryValue}`],
+        ["", false, `--ram=${expectedMemoryValue}`],
+        ["512", false, "--ram=512"],
+        [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
+        ["", true, `--ram=${expectedMemoryValueWithScaling}`],
     ];
-    for (const [input, expectedFlag] of tests) {
-        const flag = util.getMemoryFlag(input);
+    for (const [input, withScaling, expectedFlag] of tests) {
+        const features = (0, testing_utils_1.createFeatures)(withScaling ? [feature_flags_1.Feature.ScalingReservedRam] : []);
+        const flag = await util.getMemoryFlag(input, features);
         t.deepEqual(flag, expectedFlag);
     }
 });
-(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
+(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
     for (const input of ["-1", "hello!"]) {
-        t.throws(() => util.getMemoryFlag(input));
+        await t.throwsAsync(async () => await util.getMemoryFlag(input, (0, testing_utils_1.createFeatures)([])));
     }
 });
 (0, ava_1.default)("getAddSnippetsFlag() should return the correct flag", (t) => {

pr-checks/checks/scaling-reserved-ram.yml

@@ -0,0 +1,24 @@
+name: "Scaling reserved RAM"
+description: "An end-to-end integration test of a multi-language repository with the scaling_reserved_ram feature flag enabled"
+operatingSystems: ["ubuntu", "macos"]
+env:
+  CODEQL_ACTION_SCALING_RESERVED_RAM: true
+steps:
+  - uses: ./../action/init
+    id: init
+    with:
+      db-location: "${{ runner.temp }}/customDbLocation"
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+
+  - uses: ./../action/.github/actions/setup-swift
+    with:
+      codeql-path: ${{ steps.init.outputs.codeql-path }}
+
+  - name: Build code
+    shell: bash
+    run: ./build.sh
+
+  - uses: ./../action/analyze
+    id: analysis
+    with:
+      upload-database: false

src/analyze-action.ts

@@ -210,9 +210,6 @@ async function run() {
       actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
       logger
     );
-    const memory = util.getMemoryFlag(
-      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
-    );
 
     const repositoryNwo = parseRepositoryNwo(
       util.getRequiredEnvParam("GITHUB_REPOSITORY")
@@ -227,6 +224,11 @@ async function run() {
       logger
     );
 
+    const memory = await util.getMemoryFlag(
+      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"],
+      features
+    );
+
     await runAutobuildIfLegacyGoWorkflow(config, logger);
 
     dbCreationTimings = await runFinalize(

src/feature-flags.ts

@@ -39,6 +39,7 @@ export enum Feature {
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
   NewAnalysisSummaryEnabled = "new_analysis_summary_enabled",
   QaTelemetryEnabled = "qa_telemetry_enabled",
+  ScalingReservedRam = "scaling_reserved_ram",
   UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
 }
 
@@ -76,6 +77,11 @@ export const featureConfig: Record<
     minimumVersion: undefined,
     defaultValue: false,
   },
+  [Feature.ScalingReservedRam]: {
+    envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
+    minimumVersion: undefined,
+    defaultValue: false,
+  },
   [Feature.UploadFailedSarifEnabled]: {
     envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
     minimumVersion: "2.11.3",

src/init-action.ts

@@ -329,7 +329,7 @@ async function run() {
     core.exportVariable(
       "CODEQL_RAM",
       process.env["CODEQL_RAM"] ||
-        getMemoryFlagValue(getOptionalInput("ram")).toString()
+        (await getMemoryFlagValue(getOptionalInput("ram"), features)).toString()
     );
     core.exportVariable(
       "CODEQL_THREADS",

src/util.test.ts

@@ -8,8 +8,14 @@ import * as sinon from "sinon";
 
 import * as api from "./api-client";
 import { Config } from "./config-utils";
+import { Feature } from "./feature-flags";
 import { getRunnerLogger } from "./logging";
-import { getRecordingLogger, LoggedMessage, setupTests } from "./testing-utils";
+import {
+  createFeatures,
+  getRecordingLogger,
+  LoggedMessage,
+  setupTests,
+} from "./testing-utils";
 import * as util from "./util";
 
 setupTests(test);
@@ -23,25 +29,37 @@ test("getToolNames", (t) => {
   t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
 
-test("getMemoryFlag() should return the correct --ram flag", (t) => {
-  const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
-  const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
+test("getMemoryFlag() should return the correct --ram flag", async (t) => {
+  const totalMem = os.totalmem() / (1024 * 1024);
+  const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+  const scaledAmount = 0.02 * totalMem;
+  const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+  const expectedMemoryValueWithScaling = Math.floor(
+    totalMem - fixedAmount - scaledAmount
+  );
 
-  const tests: Array<[string | undefined, string]> = [
-    [undefined, `--ram=${totalMem - expectedThreshold}`],
-    ["", `--ram=${totalMem - expectedThreshold}`],
-    ["512", "--ram=512"],
+  const tests: Array<[string | undefined, boolean, string]> = [
+    [undefined, false, `--ram=${expectedMemoryValue}`],
+    ["", false, `--ram=${expectedMemoryValue}`],
+    ["512", false, "--ram=512"],
+    [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
+    ["", true, `--ram=${expectedMemoryValueWithScaling}`],
   ];
 
-  for (const [input, expectedFlag] of tests) {
-    const flag = util.getMemoryFlag(input);
+  for (const [input, withScaling, expectedFlag] of tests) {
+    const features = createFeatures(
+      withScaling ? [Feature.ScalingReservedRam] : []
+    );
+    const flag = await util.getMemoryFlag(input, features);
     t.deepEqual(flag, expectedFlag);
   }
 });
 
-test("getMemoryFlag() throws if the ram input is < 0 or NaN", (t) => {
+test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
   for (const input of ["-1", "hello!"]) {
-    t.throws(() => util.getMemoryFlag(input));
+    await t.throwsAsync(
+      async () => await util.getMemoryFlag(input, createFeatures([]))
+    );
   }
 });
 

src/util.ts

@@ -157,9 +157,21 @@ export async function withTmpDir<T>(
  * from committing too much of the available memory to CodeQL.
  * @returns number
  */
-function getSystemReservedMemoryMegaBytes(): number {
+async function getSystemReservedMemoryMegaBytes(
+  totalMemoryMegaBytes: number,
+  features: FeatureEnablement
+): Promise<number> {
   // Windows needs more memory for OS processes.
-  return 1024 * (process.platform === "win32" ? 1.5 : 1);
+  const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+
+  if (await features.getValue(Feature.ScalingReservedRam)) {
+    // Reserve an additional 2% of the total memory, since the amount used by
+    // the kernel for page tables scales with the size of physical memory.
+    const scaledAmount = 0.02 * totalMemoryMegaBytes;
+    return fixedAmount + scaledAmount;
+  } else {
+    return fixedAmount;
+  }
 }
 
 /**
@@ -169,7 +181,10 @@ function getSystemReservedMemoryMegaBytes(): number {
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-export function getMemoryFlagValue(userInput: string | undefined): number {
+export async function getMemoryFlagValue(
+  userInput: string | undefined,
+  features: FeatureEnablement
+): Promise<number> {
   let memoryToUseMegaBytes: number;
   if (userInput) {
     memoryToUseMegaBytes = Number(userInput);
@@ -179,7 +194,10 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
   } else {
     const totalMemoryBytes = os.totalmem();
     const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
-    const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
+    const reservedMemoryMegaBytes = await getSystemReservedMemoryMegaBytes(
+      totalMemoryMegaBytes,
+      features
+    );
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
   }
   return Math.floor(memoryToUseMegaBytes);
@@ -192,8 +210,12 @@ export function getMemoryFlagValue(userInput: string | undefined): number {
  *
  * @returns string
  */
-export function getMemoryFlag(userInput: string | undefined): string {
-  return `--ram=${getMemoryFlagValue(userInput)}`;
+export async function getMemoryFlag(
+  userInput: string | undefined,
+  features: FeatureEnablement
+): Promise<string> {
+  const megabytes = await getMemoryFlagValue(userInput, features);
+  return `--ram=${megabytes}`;
 }
 
 /**