Merge pull request #2112 from github/henrymercer/status-report-handle-disk-failures

Telemetry: Handle failures determining disk usage
This commit is contained in:
Henry Mercer 2024-02-02 17:31:09 +00:00 committed by GitHub
commit 2db0327171
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 66 additions and 42 deletions

6
lib/status-report.js generated
View file

@ -103,9 +103,7 @@ async function createStatusReportBase(actionName, status, actionStartedAt, diskI
job_name: jobName,
job_run_uuid: jobRunUUID,
ref,
runner_available_disk_space_bytes: diskInfo.numAvailableBytes,
runner_os: runnerOs,
runner_total_disk_space_bytes: diskInfo.numTotalBytes,
started_at: workflowStartedAt,
status,
testing_environment: testingEnvironment,
@ -113,6 +111,10 @@ async function createStatusReportBase(actionName, status, actionStartedAt, diskI
workflow_run_attempt: workflowRunAttempt,
workflow_run_id: workflowRunID,
};
if (diskInfo) {
statusReport.runner_available_disk_space_bytes = diskInfo.numAvailableBytes;
statusReport.runner_total_disk_space_bytes = diskInfo.numTotalBytes;
}
// Add optional parameters
if (cause) {
statusReport.cause = cause;

File diff suppressed because one or more lines are too long

38
lib/util.js generated
View file

@ -789,23 +789,31 @@ function prettyPrintPack(pack) {
}
exports.prettyPrintPack = prettyPrintPack;
async function checkDiskUsage(logger) {
const diskUsage = await (0, check_disk_space_1.default)(getRequiredEnvParam("GITHUB_WORKSPACE"));
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message = "The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[environment_1.EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
try {
const diskUsage = await (0, check_disk_space_1.default)(getRequiredEnvParam("GITHUB_WORKSPACE"));
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message = "The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[environment_1.EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
}
else {
logger.debug(message);
}
core.exportVariable(environment_1.EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
}
else {
logger.debug(message);
}
core.exportVariable(environment_1.EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
}
catch (error) {
if (logger) {
logger.warning(`Failed to check available disk space: ${getErrorMessage(error)}`);
}
return undefined;
}
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
}
exports.checkDiskUsage = checkDiskUsage;
/**

File diff suppressed because one or more lines are too long

View file

@ -96,7 +96,7 @@ export interface StatusReportBase {
/** Action runner hardware architecture (context runner.arch). */
runner_arch?: string;
/** Available disk space on the runner, in bytes. */
runner_available_disk_space_bytes: number;
runner_available_disk_space_bytes?: number;
/**
* Version of the runner image, for workflows running on GitHub-hosted runners. Absent otherwise.
*/
@ -106,7 +106,7 @@ export interface StatusReportBase {
/** Action runner operating system release (x.y.z from os.release()). */
runner_os_release?: string;
/** Total disk space on the runner, in bytes. */
runner_total_disk_space_bytes: number;
runner_total_disk_space_bytes?: number;
/** Time the first action started. Normally the init action. */
started_at: string;
/** State this action is currently in. */
@ -192,7 +192,7 @@ export async function createStatusReportBase(
actionName: ActionName,
status: ActionStatus,
actionStartedAt: Date,
diskInfo: DiskUsage,
diskInfo: DiskUsage | undefined,
cause?: string,
exception?: string,
): Promise<StatusReportBase> {
@ -230,9 +230,7 @@ export async function createStatusReportBase(
job_name: jobName,
job_run_uuid: jobRunUUID,
ref,
runner_available_disk_space_bytes: diskInfo.numAvailableBytes,
runner_os: runnerOs,
runner_total_disk_space_bytes: diskInfo.numTotalBytes,
started_at: workflowStartedAt,
status,
testing_environment: testingEnvironment,
@ -241,6 +239,11 @@ export async function createStatusReportBase(
workflow_run_id: workflowRunID,
};
if (diskInfo) {
statusReport.runner_available_disk_space_bytes = diskInfo.numAvailableBytes;
statusReport.runner_total_disk_space_bytes = diskInfo.numTotalBytes;
}
// Add optional parameters
if (cause) {
statusReport.cause = cause;

View file

@ -1011,26 +1011,37 @@ export interface DiskUsage {
numTotalBytes: number;
}
export async function checkDiskUsage(logger?: Logger): Promise<DiskUsage> {
const diskUsage = await checkDiskSpace(
getRequiredEnvParam("GITHUB_WORKSPACE"),
);
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message =
"The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
} else {
logger.debug(message);
export async function checkDiskUsage(
logger?: Logger,
): Promise<DiskUsage | undefined> {
try {
const diskUsage = await checkDiskSpace(
getRequiredEnvParam("GITHUB_WORKSPACE"),
);
const gbInBytes = 1024 * 1024 * 1024;
if (logger && diskUsage.free < 2 * gbInBytes) {
const message =
"The Actions runner is running low on disk space " +
`(${(diskUsage.free / gbInBytes).toPrecision(4)} GB available).`;
if (process.env[EnvVar.HAS_WARNED_ABOUT_DISK_SPACE] !== "true") {
logger.warning(message);
} else {
logger.debug(message);
}
core.exportVariable(EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
}
core.exportVariable(EnvVar.HAS_WARNED_ABOUT_DISK_SPACE, "true");
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
} catch (error) {
if (logger) {
logger.warning(
`Failed to check available disk space: ${getErrorMessage(error)}`,
);
}
return undefined;
}
return {
numAvailableBytes: diskUsage.free,
numTotalBytes: diskUsage.size,
};
}
/**