robojerk/codeql-action

Increase the default amount of RAM reserved for the OS

Browse source 
Mitigation for OOM errors (137/SIGKILL) seen by users when we overcommit the available memory.
For Unix, reserve 1GB.
For Windows, reserve 1.5GB, as the OS needs more memory and estimates inaccurately.
This commit is contained in:
Aditya Sharad 2021-02-16 15:04:38 -08:00
parent 88714e3a60
commit 4c94e29f1b
No known key found for this signature in database
GPG key ID: 66D1E528054C320C
6 changed files with 39 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

18
lib/util.js generated
View file

@ -72,9 +72,21 @@ async function withTmpDir(body) {
return result;
}
exports.withTmpDir = withTmpDir;
/**
* Gets an OS-specific amount of memory (in MB) to reserve for OS processes
* when the user doesn't explicitly specify a memory setting.
* This is a heuristic to avoid OOM errors (exit code 137 / SIGKILL)
* from committing too much of the available memory to CodeQL.
* @returns number
*/
function getSystemReservedMemoryMegaBytes() {
// Windows needs more memory for OS processes.
return 1024 * (process.platform === "win32" ? 1.5 : 1);
}
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
* specified, the total available memory will be used minus 256 MB.
* specified, the total available memory will be used minus a threshold
* reserved for the OS.
*
* @returns string
*/
@ -89,8 +101,8 @@ function getMemoryFlag(userInput) {
else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
}

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

5
lib/util.test.js generated
View file

@ -28,9 +28,10 @@ ava_1.default("getToolNames", (t) => {
});
ava_1.default("getMemoryFlag() should return the correct --ram flag", (t) => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
const tests = [
[undefined, `--ram=${totalMem - 256}`],
["", `--ram=${totalMem - 256}`],
[undefined, `--ram=${totalMem - expectedThreshold}`],
["", `--ram=${totalMem - expectedThreshold}`],
["512", "--ram=512"],
];
for (const [input, expectedFlag] of tests) {

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

5
src/util.test.ts
View file

@ -24,10 +24,11 @@ test("getToolNames", (t) => {
test("getMemoryFlag() should return the correct --ram flag", (t) => {
const totalMem = Math.floor(os.totalmem() / (1024 * 1024));
const expectedThreshold = process.platform === "win32" ? 1536 : 1024;
const tests = [
[undefined, `--ram=${totalMem - 256}`],
["", `--ram=${totalMem - 256}`],
[undefined, `--ram=${totalMem - expectedThreshold}`],
["", `--ram=${totalMem - expectedThreshold}`],
["512", "--ram=512"],
];

19
src/util.ts
View file

@ -82,9 +82,22 @@ export async function withTmpDir<T>(
return result;
}
/**
* Gets an OS-specific amount of memory (in MB) to reserve for OS processes
* when the user doesn't explicitly specify a memory setting.
* This is a heuristic to avoid OOM errors (exit code 137 / SIGKILL)
* from committing too much of the available memory to CodeQL.
* @returns number
*/
function getSystemReservedMemoryMegaBytes(): number {
// Windows needs more memory for OS processes.
return 1024 * (process.platform === "win32" ? 1.5 : 1);
}
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
* specified, the total available memory will be used minus 256 MB.
* specified, the total available memory will be used minus a threshold
* reserved for the OS.
*
* @returns string
*/
@ -98,8 +111,8 @@ export function getMemoryFlag(userInput: string | undefined): string {
} else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const systemReservedMemoryMegaBytes = 256;
memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
}