Test memory flag computation across all platforms and system RAM values

lib/util.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.prettyPrintPack = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.wrapError = exports.fixInvalidNotificationsInFile = exports.fixInvalidNotifications = exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.isInTestMode = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.initializeEnvironment = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
+exports.prettyPrintPack = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.wrapError = exports.fixInvalidNotificationsInFile = exports.fixInvalidNotifications = exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.isInTestMode = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.initializeEnvironment = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.getMemoryFlagValueForPlatform = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
@@ -105,9 +105,9 @@ exports.withTmpDir = withTmpDir;
  * from committing too much of the available memory to CodeQL.
  * @returns number
  */
-function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReservedRamEnabled) {
+function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled) {
     // Windows needs more memory for OS processes.
-    const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+    const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
     if (isScalingReservedRamEnabled) {
         // Reserve an additional 2% of the total memory, since the amount used by
         // the kernel for page tables scales with the size of physical memory.
@@ -125,7 +125,7 @@ function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReserve
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
+function getMemoryFlagValueForPlatform(userInput, totalMemoryBytes, platform, isScalingReservedRamEnabled) {
     let memoryToUseMegaBytes;
     if (userInput) {
         memoryToUseMegaBytes = Number(userInput);
@@ -134,13 +134,23 @@ function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
         }
     }
     else {
-        const totalMemoryBytes = os.totalmem();
         const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
-        const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReservedRamEnabled);
+        const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled);
         memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
     }
     return Math.floor(memoryToUseMegaBytes);
 }
+exports.getMemoryFlagValueForPlatform = getMemoryFlagValueForPlatform;
+/**
+ * Get the value of the codeql `--ram` flag as configured by the `ram` input.
+ * If no value was specified, the total available memory will be used minus a
+ * threshold reserved for the OS.
+ *
+ * @returns {number} the amount of RAM to use, in megabytes
+ */
+function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
+    return getMemoryFlagValueForPlatform(userInput, os.totalmem(), process.platform, isScalingReservedRamEnabled);
+}
 exports.getMemoryFlagValue = getMemoryFlagValue;
 /**
  * Get the codeql `--ram` flag as configured by the `ram` input. If no value was

lib/util.test.js

@@ -39,24 +39,59 @@ const util = __importStar(require("./util"));
     const toolNames = util.getToolNames(JSON.parse(input));
     t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
-(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", async (t) => {
+const GET_MEMORY_FLAG_TESTS = [
-    const totalMem = os.totalmem() / (1024 * 1024);
+    {
-    const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+        input: undefined,
-    const scaledAmount = 0.02 * totalMem;
+        totalMemoryMb: 8 * 1024,
-    const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+        platform: "linux",
-    const expectedMemoryValueWithScaling = Math.floor(totalMem - fixedAmount - scaledAmount);
+        expectedMemoryValue: 7 * 1024,
-    const tests = [
+        expectedMemoryValueWithScaling: 7004, // Math.floor(1024 * (8*0.98 - 1))
-        [undefined, false, `--ram=${expectedMemoryValue}`],
+    },
-        ["", false, `--ram=${expectedMemoryValue}`],
+    {
-        ["512", false, "--ram=512"],
+        input: undefined,
-        [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
+        totalMemoryMb: 8 * 1024,
-        ["", true, `--ram=${expectedMemoryValueWithScaling}`],
+        platform: "win32",
-    ];
+        expectedMemoryValue: 6.5 * 1024,
-    for (const [input, withScaling, expectedFlag] of tests) {
+        expectedMemoryValueWithScaling: 6492, // Math.floor(1024 * (8*0.98 - 1.5))
-        const flag = util.getMemoryFlag(input, withScaling);
+    },
-        t.deepEqual(flag, expectedFlag);
+    {
-    }
+        input: "",
-});
+        totalMemoryMb: 8 * 1024,
+        platform: "linux",
+        expectedMemoryValue: 7 * 1024,
+        expectedMemoryValueWithScaling: 7004, // Math.floor(1024 * (8*0.98 - 1))
+    },
+    {
+        input: "512",
+        totalMemoryMb: 8 * 1024,
+        platform: "linux",
+        expectedMemoryValue: 512,
+        expectedMemoryValueWithScaling: 512,
+    },
+    {
+        input: undefined,
+        totalMemoryMb: 64 * 1024,
+        platform: "linux",
+        expectedMemoryValue: 63 * 1024,
+        expectedMemoryValueWithScaling: 63201, // Math.floor(1024 * (64*0.98 - 1))
+    },
+    {
+        input: undefined,
+        totalMemoryMb: 64 * 1024,
+        platform: "win32",
+        expectedMemoryValue: 62.5 * 1024,
+        expectedMemoryValueWithScaling: 62689, // Math.floor(1024 * (64*0.98 - 1.5))
+    },
+];
+for (const { input, totalMemoryMb, platform, expectedMemoryValue, expectedMemoryValueWithScaling, } of GET_MEMORY_FLAG_TESTS) {
+    (0, ava_1.default)(`Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
+        `for ${input ?? "no user input"} on ${platform} with ${totalMemoryMb} MB total system RAM`, async (t) => {
+        for (const withScaling of [true, false]) {
+            const flag = util.getMemoryFlagValueForPlatform(input, totalMemoryMb * 1024 * 1024, platform, withScaling);
+            t.deepEqual(flag, withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue);
+        }
+    });
+}
 (0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
     for (const input of ["-1", "hello!"]) {
         t.throws(() => util.getMemoryFlag(input, false));

src/util.test.ts

@@ -19,28 +19,79 @@ test("getToolNames", (t) => {
   t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
 });
 
-test("getMemoryFlag() should return the correct --ram flag", async (t) => {
+const GET_MEMORY_FLAG_TESTS = [
-  const totalMem = os.totalmem() / (1024 * 1024);
+  {
-  const fixedAmount = process.platform === "win32" ? 1536 : 1024;
+    input: undefined,
-  const scaledAmount = 0.02 * totalMem;
+    totalMemoryMb: 8 * 1024,
-  const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
+    platform: "linux",
-  const expectedMemoryValueWithScaling = Math.floor(
+    expectedMemoryValue: 7 * 1024,
-    totalMem - fixedAmount - scaledAmount
+    expectedMemoryValueWithScaling: 7004, // Math.floor(1024 * (8*0.98 - 1))
+  },
+  {
+    input: undefined,
+    totalMemoryMb: 8 * 1024,
+    platform: "win32",
+    expectedMemoryValue: 6.5 * 1024,
+    expectedMemoryValueWithScaling: 6492, // Math.floor(1024 * (8*0.98 - 1.5))
+  },
+  {
+    input: "",
+    totalMemoryMb: 8 * 1024,
+    platform: "linux",
+    expectedMemoryValue: 7 * 1024,
+    expectedMemoryValueWithScaling: 7004, // Math.floor(1024 * (8*0.98 - 1))
+  },
+  {
+    input: "512",
+    totalMemoryMb: 8 * 1024,
+    platform: "linux",
+    expectedMemoryValue: 512,
+    expectedMemoryValueWithScaling: 512,
+  },
+  {
+    input: undefined,
+    totalMemoryMb: 64 * 1024,
+    platform: "linux",
+    expectedMemoryValue: 63 * 1024,
+    expectedMemoryValueWithScaling: 63201, // Math.floor(1024 * (64*0.98 - 1))
+  },
+  {
+    input: undefined,
+    totalMemoryMb: 64 * 1024,
+    platform: "win32",
+    expectedMemoryValue: 62.5 * 1024,
+    expectedMemoryValueWithScaling: 62689, // Math.floor(1024 * (64*0.98 - 1.5))
+  },
+];
+
+for (const {
+  input,
+  totalMemoryMb,
+  platform,
+  expectedMemoryValue,
+  expectedMemoryValueWithScaling,
+} of GET_MEMORY_FLAG_TESTS) {
+  test(
+    `Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
+      `for ${
+        input ?? "no user input"
+      } on ${platform} with ${totalMemoryMb} MB total system RAM`,
+    async (t) => {
+      for (const withScaling of [true, false]) {
+        const flag = util.getMemoryFlagValueForPlatform(
+          input,
+          totalMemoryMb * 1024 * 1024,
+          platform,
+          withScaling
+        );
+        t.deepEqual(
+          flag,
+          withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue
+        );
+      }
+    }
   );
-
+}
-  const tests: Array<[string | undefined, boolean, string]> = [
-    [undefined, false, `--ram=${expectedMemoryValue}`],
-    ["", false, `--ram=${expectedMemoryValue}`],
-    ["512", false, "--ram=512"],
-    [undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
-    ["", true, `--ram=${expectedMemoryValueWithScaling}`],
-  ];
-
-  for (const [input, withScaling, expectedFlag] of tests) {
-    const flag = util.getMemoryFlag(input, withScaling);
-    t.deepEqual(flag, expectedFlag);
-  }
-});
 
 test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
   for (const input of ["-1", "hello!"]) {

src/util.ts

@@ -153,10 +153,11 @@ export async function withTmpDir<T>(
  */
 function getSystemReservedMemoryMegaBytes(
   totalMemoryMegaBytes: number,
+  platform: string,
   isScalingReservedRamEnabled: boolean
 ): number {
   // Windows needs more memory for OS processes.
-  const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
+  const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
 
   if (isScalingReservedRamEnabled) {
     // Reserve an additional 2% of the total memory, since the amount used by
@@ -175,8 +176,10 @@ function getSystemReservedMemoryMegaBytes(
  *
  * @returns {number} the amount of RAM to use, in megabytes
  */
-export function getMemoryFlagValue(
+export function getMemoryFlagValueForPlatform(
   userInput: string | undefined,
+  totalMemoryBytes: number,
+  platform: string,
   isScalingReservedRamEnabled: boolean
 ): number {
   let memoryToUseMegaBytes: number;
@@ -186,10 +189,10 @@ export function getMemoryFlagValue(
       throw new Error(`Invalid RAM setting "${userInput}", specified.`);
     }
   } else {
-    const totalMemoryBytes = os.totalmem();
     const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
     const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(
       totalMemoryMegaBytes,
+      platform,
       isScalingReservedRamEnabled
     );
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
@@ -197,6 +200,25 @@ export function getMemoryFlagValue(
   return Math.floor(memoryToUseMegaBytes);
 }
 
+/**
+ * Get the value of the codeql `--ram` flag as configured by the `ram` input.
+ * If no value was specified, the total available memory will be used minus a
+ * threshold reserved for the OS.
+ *
+ * @returns {number} the amount of RAM to use, in megabytes
+ */
+export function getMemoryFlagValue(
+  userInput: string | undefined,
+  isScalingReservedRamEnabled: boolean
+): number {
+  return getMemoryFlagValueForPlatform(
+    userInput,
+    os.totalmem(),
+    process.platform,
+    isScalingReservedRamEnabled
+  );
+}
+
 /**
  * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
  * specified, the total available memory will be used minus a threshold