robojerk/codeql-action

Merge pull request #1796 from github/henrymercer/scaling-ram-larger-runners-only

Browse source 
Scale the amount of reserved RAM on large runners only
This commit is contained in:
Henry Mercer 2023-07-24 12:47:48 +01:00 • committed by GitHub
commit 7b6664fa89
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 172 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

26
lib/util.js generated
View file

@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.prettyPrintPack = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.wrapError = exports.fixInvalidNotificationsInFile = exports.fixInvalidNotifications = exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.isInTestMode = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.initializeEnvironment = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
exports.prettyPrintPack = exports.getMlPoweredJsQueriesPack = exports.ML_POWERED_JS_QUERIES_PACK_NAME = exports.wrapError = exports.fixInvalidNotificationsInFile = exports.fixInvalidNotifications = exports.parseMatrixInput = exports.isHostedRunner = exports.checkForTimeout = exports.withTimeout = exports.tryGetFolderBytes = exports.listFolder = exports.doesDirectoryExist = exports.isInTestMode = exports.supportExpectDiscardedCache = exports.isGoodVersion = exports.delay = exports.bundleDb = exports.codeQlVersionAbove = exports.getCachedCodeQlVersion = exports.cacheCodeQlVersion = exports.isHTTPError = exports.UserError = exports.HTTPError = exports.getRequiredEnvParam = exports.initializeEnvironment = exports.assertNever = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.getMemoryFlagValueForPlatform = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.DEFAULT_DEBUG_DATABASE_NAME = exports.DEFAULT_DEBUG_ARTIFACT_NAME = exports.GITHUB_DOTCOM_URL = void 0;
const fs = __importStar(require("fs"));
const os = __importStar(require("os"));
const path = __importStar(require("path"));
@ -105,13 +105,13 @@ exports.withTmpDir = withTmpDir;
* from committing too much of the available memory to CodeQL.
* @returns number
*/
function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReservedRamEnabled) {
function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled) {
// Windows needs more memory for OS processes.
const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
if (isScalingReservedRamEnabled) {
// Reserve an additional 2% of the total memory, since the amount used by
// Reserve an additional 2.5% of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount = 0.02 * totalMemoryMegaBytes;
const scaledAmount = 0.025 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
}
else {
@ -125,7 +125,7 @@ function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReserve
*
* @returns {number} the amount of RAM to use, in megabytes
*/
function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
function getMemoryFlagValueForPlatform(userInput, totalMemoryBytes, platform, isScalingReservedRamEnabled) {
let memoryToUseMegaBytes;
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
@ -134,13 +134,23 @@ function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
}
}
else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, isScalingReservedRamEnabled);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled);
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
return Math.floor(memoryToUseMegaBytes);
}
exports.getMemoryFlagValueForPlatform = getMemoryFlagValueForPlatform;
/**
* Get the value of the codeql `--ram` flag as configured by the `ram` input.
* If no value was specified, the total available memory will be used minus a
* threshold reserved for the OS.
*
* @returns {number} the amount of RAM to use, in megabytes
*/
function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
return getMemoryFlagValueForPlatform(userInput, os.totalmem(), process.platform, isScalingReservedRamEnabled);
}
exports.getMemoryFlagValue = getMemoryFlagValue;
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

71
lib/util.test.js generated
View file

@ -39,24 +39,59 @@ const util = __importStar(require("./util"));
const toolNames = util.getToolNames(JSON.parse(input));
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
(0, ava_1.default)("getMemoryFlag() should return the correct --ram flag", async (t) => {
const totalMem = os.totalmem() / (1024 * 1024);
const fixedAmount = process.platform === "win32" ? 1536 : 1024;
const scaledAmount = 0.02 * totalMem;
const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
const expectedMemoryValueWithScaling = Math.floor(totalMem - fixedAmount - scaledAmount);
const tests = [
[undefined, false, `--ram=${expectedMemoryValue}`],
["", false, `--ram=${expectedMemoryValue}`],
["512", false, "--ram=512"],
[undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
["", true, `--ram=${expectedMemoryValueWithScaling}`],
];
for (const [input, withScaling, expectedFlag] of tests) {
const flag = util.getMemoryFlag(input, withScaling);
t.deepEqual(flag, expectedFlag);
}
});
const GET_MEMORY_FLAG_TESTS = [
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "win32",
expectedMemoryValue: 6.5 * 1024,
expectedMemoryValueWithScaling: 6.5 * 1024,
},
{
input: "",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: "512",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 512,
expectedMemoryValueWithScaling: 512,
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 63078, // Math.floor(1024 * (64 - 1 - 0.025 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "win32",
expectedMemoryValue: 62.5 * 1024,
expectedMemoryValueWithScaling: 62566, // Math.floor(1024 * (64 - 1.5 - 0.025 * (64 - 8)))
},
];
for (const { input, totalMemoryMb, platform, expectedMemoryValue, expectedMemoryValueWithScaling, } of GET_MEMORY_FLAG_TESTS) {
(0, ava_1.default)(`Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
`for ${input ?? "no user input"} on ${platform} with ${totalMemoryMb} MB total system RAM`, async (t) => {
for (const withScaling of [true, false]) {
const flag = util.getMemoryFlagValueForPlatform(input, totalMemoryMb * 1024 * 1024, platform, withScaling);
t.deepEqual(flag, withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue);
}
});
}
(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
for (const input of ["-1", "hello!"]) {
t.throws(() => util.getMemoryFlag(input, false));

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

93
src/util.test.ts
View file

@ -19,28 +19,79 @@ test("getToolNames", (t) => {
t.deepEqual(toolNames, ["CodeQL command-line toolchain", "ESLint"]);
});
test("getMemoryFlag() should return the correct --ram flag", async (t) => {
const totalMem = os.totalmem() / (1024 * 1024);
const fixedAmount = process.platform === "win32" ? 1536 : 1024;
const scaledAmount = 0.02 * totalMem;
const expectedMemoryValue = Math.floor(totalMem - fixedAmount);
const expectedMemoryValueWithScaling = Math.floor(
totalMem - fixedAmount - scaledAmount
const GET_MEMORY_FLAG_TESTS = [
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "win32",
expectedMemoryValue: 6.5 * 1024,
expectedMemoryValueWithScaling: 6.5 * 1024,
},
{
input: "",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: "512",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 512,
expectedMemoryValueWithScaling: 512,
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 63078, // Math.floor(1024 * (64 - 1 - 0.025 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "win32",
expectedMemoryValue: 62.5 * 1024,
expectedMemoryValueWithScaling: 62566, // Math.floor(1024 * (64 - 1.5 - 0.025 * (64 - 8)))
},
];
for (const {
input,
totalMemoryMb,
platform,
expectedMemoryValue,
expectedMemoryValueWithScaling,
} of GET_MEMORY_FLAG_TESTS) {
test(
`Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
`for ${
input ?? "no user input"
} on ${platform} with ${totalMemoryMb} MB total system RAM`,
async (t) => {
for (const withScaling of [true, false]) {
const flag = util.getMemoryFlagValueForPlatform(
input,
totalMemoryMb * 1024 * 1024,
platform,
withScaling
);
t.deepEqual(
flag,
withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue
);
}
}
);
const tests: Array<[string | undefined, boolean, string]> = [
[undefined, false, `--ram=${expectedMemoryValue}`],
["", false, `--ram=${expectedMemoryValue}`],
["512", false, "--ram=512"],
[undefined, true, `--ram=${expectedMemoryValueWithScaling}`],
["", true, `--ram=${expectedMemoryValueWithScaling}`],
];
for (const [input, withScaling, expectedFlag] of tests) {
const flag = util.getMemoryFlag(input, withScaling);
t.deepEqual(flag, expectedFlag);
}
});
}
test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
for (const input of ["-1", "hello!"]) {

32
src/util.ts
View file

@ -153,15 +153,16 @@ export async function withTmpDir<T>(
*/
function getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes: number,
platform: string,
isScalingReservedRamEnabled: boolean
): number {
// Windows needs more memory for OS processes.
const fixedAmount = 1024 * (process.platform === "win32" ? 1.5 : 1);
const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
if (isScalingReservedRamEnabled) {
// Reserve an additional 2% of the total memory, since the amount used by
// Reserve an additional 2.5% of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount = 0.02 * totalMemoryMegaBytes;
const scaledAmount = 0.025 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
} else {
return fixedAmount;
@ -175,8 +176,10 @@ function getSystemReservedMemoryMegaBytes(
*
* @returns {number} the amount of RAM to use, in megabytes
*/
export function getMemoryFlagValue(
export function getMemoryFlagValueForPlatform(
userInput: string | undefined,
totalMemoryBytes: number,
platform: string,
isScalingReservedRamEnabled: boolean
): number {
let memoryToUseMegaBytes: number;
@ -186,10 +189,10 @@ export function getMemoryFlagValue(
throw new Error(`Invalid RAM setting "${userInput}", specified.`);
}
} else {
const totalMemoryBytes = os.totalmem();
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes,
platform,
isScalingReservedRamEnabled
);
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
@ -197,6 +200,25 @@ export function getMemoryFlagValue(
return Math.floor(memoryToUseMegaBytes);
}
/**
* Get the value of the codeql `--ram` flag as configured by the `ram` input.
* If no value was specified, the total available memory will be used minus a
* threshold reserved for the OS.
*
* @returns {number} the amount of RAM to use, in megabytes
*/
export function getMemoryFlagValue(
userInput: string | undefined,
isScalingReservedRamEnabled: boolean
): number {
return getMemoryFlagValueForPlatform(
userInput,
os.totalmem(),
process.platform,
isScalingReservedRamEnabled
);
}
/**
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
* specified, the total available memory will be used minus a threshold