robojerk/codeql-action

Remove scaling reserved RAM feature flag

Browse source
This commit is contained in:
Henry Mercer 2023-09-05 14:30:56 +02:00
parent 574dbbc517
commit 10389f671b
15 changed files with 58 additions and 123 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/analyze-action.js generated
View file

@ -163,7 +163,7 @@ async function run() {
const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
const gitHubVersion = await (0, api_client_1.getGitHubVersion)();
const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"], await features.getValue(feature_flags_1.Feature.ScalingReservedRamEnabled));
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, features);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/feature-flags.js generated
View file

@ -65,7 +65,6 @@ var Feature;
Feature["LanguageBaselineConfigEnabled"] = "language_baseline_config_enabled";
Feature["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
Feature["QaTelemetryEnabled"] = "qa_telemetry_enabled";
Feature["ScalingReservedRamEnabled"] = "scaling_reserved_ram_enabled";
Feature["UploadFailedSarifEnabled"] = "upload_failed_sarif_enabled";
})(Feature || (exports.Feature = Feature = {}));
exports.featureConfig = {
@ -114,11 +113,6 @@ exports.featureConfig = {
minimumVersion: undefined,
defaultValue: false,
},
[Feature.ScalingReservedRamEnabled]: {
envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
minimumVersion: undefined,
defaultValue: true,
},
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",

2
lib/feature-flags.js.map
View file

File diff suppressed because one or more lines are too long

2
lib/init-action.js generated
View file

@ -173,7 +173,7 @@ async function run() {
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
// for details.
core.exportVariable("CODEQL_RAM", process.env["CODEQL_RAM"] ||
(0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram"), await features.getValue(feature_flags_1.Feature.ScalingReservedRamEnabled)).toString());
(0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram")).toString());
core.exportVariable("CODEQL_THREADS", (0, util_1.getThreadsFlagValue)((0, actions_util_1.getOptionalInput)("threads"), logger).toString());
// Disable Kotlin extractor if feature flag set
if (await features.getValue(feature_flags_1.Feature.DisableKotlinAnalysisEnabled)) {

2
lib/init-action.js.map
View file

File diff suppressed because one or more lines are too long

28
lib/util.js generated
View file

@ -110,19 +110,13 @@ exports.withTmpDir = withTmpDir;
* from committing too much of the available memory to CodeQL.
* @returns number
*/
function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled) {
function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform) {
// Windows needs more memory for OS processes.
const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
if (isScalingReservedRamEnabled) {
// Reserve an additional percentage of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount = getReservedRamScaleFactor() *
Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
}
else {
return fixedAmount;
}
// Reserve an additional percentage of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount = getReservedRamScaleFactor() * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
}
function getReservedRamScaleFactor() {
const envVar = Number.parseInt(process.env[environment_1.EnvVar.SCALING_RESERVED_RAM_PERCENTAGE] || "", 10);
@ -138,7 +132,7 @@ function getReservedRamScaleFactor() {
*
* @returns {number} the amount of RAM to use, in megabytes
*/
function getMemoryFlagValueForPlatform(userInput, totalMemoryBytes, platform, isScalingReservedRamEnabled) {
function getMemoryFlagValueForPlatform(userInput, totalMemoryBytes, platform) {
let memoryToUseMegaBytes;
if (userInput) {
memoryToUseMegaBytes = Number(userInput);
@ -148,7 +142,7 @@ function getMemoryFlagValueForPlatform(userInput, totalMemoryBytes, platform, is
}
else {
const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScalingReservedRamEnabled);
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform);
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
return Math.floor(memoryToUseMegaBytes);
@ -161,8 +155,8 @@ exports.getMemoryFlagValueForPlatform = getMemoryFlagValueForPlatform;
*
* @returns {number} the amount of RAM to use, in megabytes
*/
function getMemoryFlagValue(userInput, isScalingReservedRamEnabled) {
return getMemoryFlagValueForPlatform(userInput, os.totalmem(), process.platform, isScalingReservedRamEnabled);
function getMemoryFlagValue(userInput) {
return getMemoryFlagValueForPlatform(userInput, os.totalmem(), process.platform);
}
exports.getMemoryFlagValue = getMemoryFlagValue;
/**
@ -172,8 +166,8 @@ exports.getMemoryFlagValue = getMemoryFlagValue;
*
* @returns string
*/
function getMemoryFlag(userInput, isScalingReservedRamEnabled) {
const megabytes = getMemoryFlagValue(userInput, isScalingReservedRamEnabled);
function getMemoryFlag(userInput) {
const megabytes = getMemoryFlagValue(userInput);
return `--ram=${megabytes}`;
}
exports.getMemoryFlag = getMemoryFlag;

2
lib/util.js.map
View file

File diff suppressed because one or more lines are too long

30
lib/util.test.js generated
View file

@ -46,68 +46,58 @@ const GET_MEMORY_FLAG_TESTS = [
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "win32",
expectedMemoryValue: 6.5 * 1024,
expectedMemoryValueWithScaling: 6.5 * 1024,
},
{
input: "",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: "512",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 512,
expectedMemoryValueWithScaling: 512,
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
expectedMemoryValue: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "win32",
expectedMemoryValue: 62.5 * 1024,
expectedMemoryValueWithScaling: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
expectedMemoryValue: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 58777,
expectedMemoryValue: 58777,
reservedPercentageValue: "10",
},
];
for (const { input, totalMemoryMb, platform, expectedMemoryValue, expectedMemoryValueWithScaling, reservedPercentageValue, } of GET_MEMORY_FLAG_TESTS) {
(0, ava_1.default)(`Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
`for ${input ?? "no user input"} on ${platform} with ${totalMemoryMb} MB total system RAM${reservedPercentageValue
? ` and reserved percentage env var set to ${reservedPercentageValue}`
: ""}`, async (t) => {
for (const { input, totalMemoryMb, platform, expectedMemoryValue, reservedPercentageValue, } of GET_MEMORY_FLAG_TESTS) {
(0, ava_1.default)(`Memory flag value is ${expectedMemoryValue} for ${input ?? "no user input"} on ${platform} with ${totalMemoryMb} MB total system RAM${reservedPercentageValue
? ` and reserved percentage env var set to ${reservedPercentageValue}`
: ""}`, async (t) => {
process.env[environment_1.EnvVar.SCALING_RESERVED_RAM_PERCENTAGE] =
reservedPercentageValue || undefined;
for (const withScaling of [true, false]) {
const flag = util.getMemoryFlagValueForPlatform(input, totalMemoryMb * 1024 * 1024, platform, withScaling);
t.deepEqual(flag, withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue);
}
const flag = util.getMemoryFlagValueForPlatform(input, totalMemoryMb * 1024 * 1024, platform);
t.deepEqual(flag, expectedMemoryValue);
});
}
(0, ava_1.default)("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
for (const input of ["-1", "hello!"]) {
t.throws(() => util.getMemoryFlag(input, false));
t.throws(() => util.getMemoryFlag(input));
}
});
(0, ava_1.default)("getAddSnippetsFlag() should return the correct flag", (t) => {

2
lib/util.test.js.map
View file

File diff suppressed because one or more lines are too long

3
src/analyze-action.ts
View file

@ -19,7 +19,7 @@ import { getCodeQL } from "./codeql";
import { Config, getConfig, getMlPoweredJsQueriesStatus } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { EnvVar } from "./environment";
import { Feature, Features } from "./feature-flags";
import { Features } from "./feature-flags";
import { Language } from "./languages";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
@ -233,7 +233,6 @@ async function run() {
const memory = util.getMemoryFlag(
actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"],
await features.getValue(Feature.ScalingReservedRamEnabled),
);
await runAutobuildIfLegacyGoWorkflow(config, logger);

6
src/feature-flags.ts
View file

@ -63,7 +63,6 @@ export enum Feature {
LanguageBaselineConfigEnabled = "language_baseline_config_enabled",
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
QaTelemetryEnabled = "qa_telemetry_enabled",
ScalingReservedRamEnabled = "scaling_reserved_ram_enabled",
UploadFailedSarifEnabled = "upload_failed_sarif_enabled",
}
@ -116,11 +115,6 @@ export const featureConfig: Record<
minimumVersion: undefined,
defaultValue: false,
},
[Feature.ScalingReservedRamEnabled]: {
envVar: "CODEQL_ACTION_SCALING_RESERVED_RAM",
minimumVersion: undefined,
defaultValue: true,
},
[Feature.UploadFailedSarifEnabled]: {
envVar: "CODEQL_ACTION_UPLOAD_FAILED_SARIF",
minimumVersion: "2.11.3",

5
src/init-action.ts
View file

@ -332,10 +332,7 @@ async function run() {
core.exportVariable(
"CODEQL_RAM",
process.env["CODEQL_RAM"] ||
getMemoryFlagValue(
getOptionalInput("ram"),
await features.getValue(Feature.ScalingReservedRamEnabled),
).toString(),
getMemoryFlagValue(getOptionalInput("ram")).toString(),
);
core.exportVariable(
"CODEQL_THREADS",

58
src/util.test.ts
View file

@ -26,49 +26,42 @@ const GET_MEMORY_FLAG_TESTS = [
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: undefined,
totalMemoryMb: 8 * 1024,
platform: "win32",
expectedMemoryValue: 6.5 * 1024,
expectedMemoryValueWithScaling: 6.5 * 1024,
},
{
input: "",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 7 * 1024,
expectedMemoryValueWithScaling: 7 * 1024,
},
{
input: "512",
totalMemoryMb: 8 * 1024,
platform: "linux",
expectedMemoryValue: 512,
expectedMemoryValueWithScaling: 512,
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
expectedMemoryValue: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "win32",
expectedMemoryValue: 62.5 * 1024,
expectedMemoryValueWithScaling: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
expectedMemoryValue: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
},
{
input: undefined,
totalMemoryMb: 64 * 1024,
platform: "linux",
expectedMemoryValue: 63 * 1024,
expectedMemoryValueWithScaling: 58777, // Math.floor(1024 * (64 - 1 - 0.1 * (64 - 8)))
expectedMemoryValue: 58777, // Math.floor(1024 * (64 - 1 - 0.1 * (64 - 8)))
reservedPercentageValue: "10",
},
];
@ -78,40 +71,29 @@ for (const {
totalMemoryMb,
platform,
expectedMemoryValue,
expectedMemoryValueWithScaling,
reservedPercentageValue,
} of GET_MEMORY_FLAG_TESTS) {
test(
`Memory flag value is ${expectedMemoryValue} without scaling and ${expectedMemoryValueWithScaling} with scaling ` +
`for ${
input ?? "no user input"
} on ${platform} with ${totalMemoryMb} MB total system RAM${
reservedPercentageValue
? ` and reserved percentage env var set to ${reservedPercentageValue}`
: ""
}`,
async (t) => {
process.env[EnvVar.SCALING_RESERVED_RAM_PERCENTAGE] =
reservedPercentageValue || undefined;
for (const withScaling of [true, false]) {
const flag = util.getMemoryFlagValueForPlatform(
input,
totalMemoryMb * 1024 * 1024,
platform,
withScaling,
);
t.deepEqual(
flag,
withScaling ? expectedMemoryValueWithScaling : expectedMemoryValue,
);
}
},
);
test(`Memory flag value is ${expectedMemoryValue} for ${
input ?? "no user input"
} on ${platform} with ${totalMemoryMb} MB total system RAM${
reservedPercentageValue
? ` and reserved percentage env var set to ${reservedPercentageValue}`
: ""
}`, async (t) => {
process.env[EnvVar.SCALING_RESERVED_RAM_PERCENTAGE] =
reservedPercentageValue || undefined;
const flag = util.getMemoryFlagValueForPlatform(
input,
totalMemoryMb * 1024 * 1024,
platform,
);
t.deepEqual(flag, expectedMemoryValue);
});
}
test("getMemoryFlag() throws if the ram input is < 0 or NaN", async (t) => {
for (const input of ["-1", "hello!"]) {
t.throws(() => util.getMemoryFlag(input, false));
t.throws(() => util.getMemoryFlag(input));
}
});

31
src/util.ts
View file

@ -160,21 +160,15 @@ export async function withTmpDir<T>(
function getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes: number,
platform: string,
isScalingReservedRamEnabled: boolean,
): number {
// Windows needs more memory for OS processes.
const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
if (isScalingReservedRamEnabled) {
// Reserve an additional percentage of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount =
getReservedRamScaleFactor() *
Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
} else {
return fixedAmount;
}
// Reserve an additional percentage of the amount of memory above 8 GB, since the amount used by
// the kernel for page tables scales with the size of physical memory.
const scaledAmount =
getReservedRamScaleFactor() * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
return fixedAmount + scaledAmount;
}
function getReservedRamScaleFactor(): number {
@ -199,7 +193,6 @@ export function getMemoryFlagValueForPlatform(
userInput: string | undefined,
totalMemoryBytes: number,
platform: string,
isScalingReservedRamEnabled: boolean,
): number {
let memoryToUseMegaBytes: number;
if (userInput) {
@ -212,7 +205,6 @@ export function getMemoryFlagValueForPlatform(
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes(
totalMemoryMegaBytes,
platform,
isScalingReservedRamEnabled,
);
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
}
@ -226,15 +218,11 @@ export function getMemoryFlagValueForPlatform(
*
* @returns {number} the amount of RAM to use, in megabytes
*/
export function getMemoryFlagValue(
userInput: string | undefined,
isScalingReservedRamEnabled: boolean,
): number {
export function getMemoryFlagValue(userInput: string | undefined): number {
return getMemoryFlagValueForPlatform(
userInput,
os.totalmem(),
process.platform,
isScalingReservedRamEnabled,
);
}
@ -245,11 +233,8 @@ export function getMemoryFlagValue(
*
* @returns string
*/
export function getMemoryFlag(
userInput: string | undefined,
isScalingReservedRamEnabled: boolean,
): string {
const megabytes = getMemoryFlagValue(userInput, isScalingReservedRamEnabled);
export function getMemoryFlag(userInput: string | undefined): string {
const megabytes = getMemoryFlagValue(userInput);
return `--ram=${megabytes}`;
}