robojerk/codeql-action

Merge remote-tracking branch 'upstream/main' into aeisenberg/better-error-message

Browse source
This commit is contained in:
Andrew Eisenberg 2022-08-23 09:29:19 -07:00
commit ac92a02de7
159 changed files with 2666 additions and 445 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/analysis-paths.test.ts
View file

@ -30,6 +30,7 @@ test("emptyPaths", async (t) => {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
@ -60,6 +61,7 @@ test("nonEmptyPaths", async (t) => {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
@ -93,6 +95,7 @@ test("exclude temp dir", async (t) => {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
analysisPaths.includeAndExcludeAnalysisPaths(config);
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);

1
src/analyze-action-env.test.ts
View file

@ -42,6 +42,7 @@ test("analyze action with RAM & threads from environment variables", async (t) =
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
optionalInputStub.withArgs("expect-error").returns("false");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
setupActionsVars(tmpDir, tmpDir);
mockFeatureFlagApiEndpoint(200, {});

1
src/analyze-action-input.test.ts
View file

@ -42,6 +42,7 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
requiredInputStub.withArgs("upload-database").returns("false");
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
optionalInputStub.withArgs("cleanup-level").returns("none");
optionalInputStub.withArgs("expect-error").returns("false");
sinon.stub(util, "getGitHubVersion").resolves(gitHubVersion);
setupActionsVars(tmpDir, tmpDir);
mockFeatureFlagApiEndpoint(200, {});

125
src/analyze-action.ts
View file

@ -1,3 +1,6 @@
// We need to import `performance` on Node 12
import { performance } from "perf_hooks";
import * as core from "@actions/core";
import * as actionsUtil from "./actions-util";
@ -13,9 +16,9 @@ import { getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { GitHubFeatureFlags } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { uploadTrapCaches } from "./trap-caching";
import { getTotalCacheSize, uploadTrapCaches } from "./trap-caching";
import * as upload_lib from "./upload-lib";
import { UploadResult } from "./upload-lib";
import * as util from "./util";
@ -31,11 +34,21 @@ interface FinishStatusReport
extends actionsUtil.StatusReportBase,
AnalysisStatusReport {}
interface FinishWithTrapUploadStatusReport extends FinishStatusReport {
/** Size of TRAP caches that we uploaded, in bytes. */
trap_cache_upload_size_bytes: number;
/** Time taken to upload TRAP caches, in milliseconds. */
trap_cache_upload_duration_ms: number;
}
export async function sendStatusReport(
startedAt: Date,
config: Config | undefined,
stats: AnalysisStatusReport | undefined,
error?: Error
error: Error | undefined,
trapCacheUploadTime: number | undefined,
didUploadTrapCaches: boolean,
logger: Logger
) {
const status = actionsUtil.getActionsStatus(
error,
@ -58,7 +71,26 @@ export async function sendStatusReport(
: {}),
...(stats || {}),
};
await actionsUtil.sendStatusReport(statusReport);
if (config && didUploadTrapCaches) {
const trapCacheUploadStatusReport: FinishWithTrapUploadStatusReport = {
...statusReport,
trap_cache_upload_duration_ms: Math.round(trapCacheUploadTime || 0),
trap_cache_upload_size_bytes: Math.round(
await getTotalCacheSize(config.trapCaches, logger)
),
};
await actionsUtil.sendStatusReport(trapCacheUploadStatusReport);
} else {
await actionsUtil.sendStatusReport(statusReport);
}
}
// `expect-error` should only be set to a non-false value by the CodeQL Action PR checks.
function hasBadExpectErrorInput(): boolean {
return (
actionsUtil.getOptionalInput("expect-error") !== "false" &&
!util.isInTestMode()
);
}
async function run() {
@ -66,9 +98,12 @@ async function run() {
let uploadResult: UploadResult | undefined = undefined;
let runStats: QueriesStatusReport | undefined = undefined;
let config: Config | undefined = undefined;
let trapCacheUploadTime: number | undefined = undefined;
let didUploadTrapCaches = false;
util.initializeEnvironment(util.Mode.actions, pkg.version);
await util.checkActionVersion(pkg.version);
const logger = getActionsLogger();
try {
if (
!(await actionsUtil.sendStatusReport(
@ -81,13 +116,19 @@ async function run() {
) {
return;
}
const logger = getActionsLogger();
config = await getConfig(actionsUtil.getTemporaryDirectory(), logger);
if (config === undefined) {
throw new Error(
"Config file could not be found at expected location. Has the 'init' action been called?"
);
}
if (hasBadExpectErrorInput()) {
throw new Error(
"`expect-error` input parameter is for internal use only. It should only be set by codeql-action or a fork."
);
}
await util.enrichEnvironment(
util.Mode.actions,
await getCodeQL(config.codeQLCmd)
@ -163,8 +204,10 @@ async function run() {
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
// Possibly upload the TRAP caches for later re-use
const trapCacheUploadStartTime = performance.now();
const codeql = await getCodeQL(config.codeQLCmd);
await uploadTrapCaches(codeql, config, logger);
didUploadTrapCaches = await uploadTrapCaches(codeql, config, logger);
trapCacheUploadTime = performance.now() - trapCacheUploadStartTime;
// We don't upload results in test mode, so don't wait for processing
if (util.isInTestMode()) {
@ -180,31 +223,83 @@ async function run() {
getActionsLogger()
);
}
// If we did not throw an error yet here, but we expect one, throw it.
if (actionsUtil.getOptionalInput("expect-error") === "true") {
core.setFailed(
`expect-error input was set to true but no error was thrown.`
);
}
} catch (origError) {
const error =
origError instanceof Error ? origError : new Error(String(origError));
core.setFailed(error.message);
if (
actionsUtil.getOptionalInput("expect-error") !== "true" ||
hasBadExpectErrorInput()
) {
core.setFailed(error.message);
}
console.log(error);
if (error instanceof CodeQLAnalysisError) {
const stats = { ...error.queriesStatusReport };
await sendStatusReport(startedAt, config, stats, error);
await sendStatusReport(
startedAt,
config,
stats,
error,
trapCacheUploadTime,
didUploadTrapCaches,
logger
);
} else {
await sendStatusReport(startedAt, config, undefined, error);
await sendStatusReport(
startedAt,
config,
undefined,
error,
trapCacheUploadTime,
didUploadTrapCaches,
logger
);
}
return;
}
if (runStats && uploadResult) {
await sendStatusReport(startedAt, config, {
...runStats,
...uploadResult.statusReport,
});
await sendStatusReport(
startedAt,
config,
{
...runStats,
...uploadResult.statusReport,
},
undefined,
trapCacheUploadTime,
didUploadTrapCaches,
logger
);
} else if (runStats) {
await sendStatusReport(startedAt, config, { ...runStats });
await sendStatusReport(
startedAt,
config,
{ ...runStats },
undefined,
trapCacheUploadTime,
didUploadTrapCaches,
logger
);
} else {
await sendStatusReport(startedAt, config, undefined);
await sendStatusReport(
startedAt,
config,
undefined,
undefined,
trapCacheUploadTime,
didUploadTrapCaches,
logger
);
}
}

2
src/analyze.test.ts
View file

@ -124,6 +124,7 @@ test("status report fields and search path setting", async (t) => {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
@ -452,6 +453,7 @@ const stubConfig: Config = {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
for (const options of [

2
src/api-compatibility.json
View file

@ -1 +1 @@
{"maximumVersion": "3.6", "minimumVersion": "3.2"}
{"maximumVersion": "3.7", "minimumVersion": "3.2"}

271
src/codeql.test.ts
View file

@ -9,10 +9,11 @@ import * as yaml from "js-yaml";
import nock from "nock";
import * as sinon from "sinon";
import { GitHubApiDetails } from "./api-client";
import * as codeql from "./codeql";
import { AugmentationProperties, Config } from "./config-utils";
import * as defaults from "./defaults.json";
import { createFeatureFlags, FeatureFlag } from "./feature-flags";
import { createFeatureFlags, FeatureFlag, FeatureFlags } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import { setupTests, setupActionsVars } from "./testing-utils";
@ -60,9 +61,58 @@ test.beforeEach(() => {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
});
async function mockApiAndSetupCodeQL({
apiDetails,
featureFlags,
isPinned,
tmpDir,
toolsInput,
version,
}: {
apiDetails?: GitHubApiDetails;
featureFlags?: FeatureFlags;
isPinned?: boolean;
tmpDir: string;
toolsInput?: { input?: string };
version: string;
}) {
const platform =
process.platform === "win32"
? "win64"
: process.platform === "linux"
? "linux64"
: "osx64";
const baseUrl = apiDetails?.url ?? "https://example.com";
const relativeUrl = apiDetails
? `/github/codeql-action/releases/download/${version}/codeql-bundle-${platform}.tar.gz`
: `/download/codeql-bundle-${version}/codeql-bundle.tar.gz`;
nock(baseUrl)
.get(relativeUrl)
.replyWithFile(
200,
path.join(
__dirname,
`/../src/testdata/codeql-bundle${isPinned ? "-pinned" : ""}.tar.gz`
)
);
await codeql.setupCodeQL(
toolsInput ? toolsInput.input : `${baseUrl}${relativeUrl}`,
apiDetails ?? sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
featureFlags ?? createFeatureFlags([]),
getRunnerLogger(true),
false
);
}
test("download codeql bundle cache", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@ -72,28 +122,11 @@ test("download codeql bundle cache", async (t) => {
for (let i = 0; i < versions.length; i++) {
const version = versions[i];
nock("https://example.com")
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
await mockApiAndSetupCodeQL({ version, tmpDir });
t.assert(toolcache.find("CodeQL", `0.0.0-${version}`));
}
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, 2);
t.is(toolcache.findAllVersions("CodeQL").length, 2);
});
});
@ -101,40 +134,14 @@ test("download codeql bundle cache explicitly requested with pinned different ve
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
nock("https://example.com")
.get(`/download/codeql-bundle-20200601/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
await codeql.setupCodeQL(
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
await mockApiAndSetupCodeQL({
version: "20200601",
isPinned: true,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
});
t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
nock("https://example.com")
.get(`/download/codeql-bundle-20200610/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
"https://example.com/download/codeql-bundle-20200610/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
await mockApiAndSetupCodeQL({ version: "20200610", tmpDir });
t.assert(toolcache.find("CodeQL", "0.0.0-20200610"));
});
});
@ -143,21 +150,11 @@ test("don't download codeql bundle cache with pinned different version cached",
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
nock("https://example.com")
.get(`/download/codeql-bundle-20200601/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
await codeql.setupCodeQL(
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
await mockApiAndSetupCodeQL({
version: "20200601",
isPinned: true,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
});
t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
@ -166,6 +163,7 @@ test("don't download codeql bundle cache with pinned different version cached",
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
createFeatureFlags([]),
getRunnerLogger(true),
false
);
@ -180,47 +178,16 @@ test("download codeql bundle cache with different version cached (not pinned)",
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
nock("https://example.com")
.get(`/download/codeql-bundle-20200601/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
await mockApiAndSetupCodeQL({ version: "20200601", tmpDir });
t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
const platform =
process.platform === "win32"
? "win64"
: process.platform === "linux"
? "linux64"
: "osx64";
nock("https://github.com")
.get(
`/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle-${platform}.tar.gz`
)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
undefined,
sampleApiDetails,
await mockApiAndSetupCodeQL({
version: defaults.bundleVersion,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
apiDetails: sampleApiDetails,
toolsInput: { input: undefined },
});
const cachedVersions = toolcache.findAllVersions("CodeQL");
@ -232,48 +199,20 @@ test('download codeql bundle cache with pinned different version cached if "late
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
nock("https://example.com")
.get(`/download/codeql-bundle-20200601/codeql-bundle.tar.gz`)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle-pinned.tar.gz`)
);
await codeql.setupCodeQL(
"https://example.com/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
sampleApiDetails,
await mockApiAndSetupCodeQL({
version: "20200601",
isPinned: true,
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
});
t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
const platform =
process.platform === "win32"
? "win64"
: process.platform === "linux"
? "linux64"
: "osx64";
nock("https://github.com")
.get(
`/github/codeql-action/releases/download/${defaults.bundleVersion}/codeql-bundle-${platform}.tar.gz`
)
.replyWithFile(
200,
path.join(__dirname, `/../src/testdata/codeql-bundle.tar.gz`)
);
await codeql.setupCodeQL(
"latest",
sampleApiDetails,
await mockApiAndSetupCodeQL({
version: defaults.bundleVersion,
apiDetails: sampleApiDetails,
toolsInput: { input: "latest" },
tmpDir,
util.GitHubVariant.DOTCOM,
getRunnerLogger(true),
false
);
});
const cachedVersions = toolcache.findAllVersions("CodeQL");
@ -281,6 +220,57 @@ test('download codeql bundle cache with pinned different version cached if "late
});
});
const TOOLCACHE_BYPASS_TEST_CASES: Array<
[boolean, string | undefined, boolean]
> = [
[true, undefined, true],
[false, undefined, false],
[
true,
"https://github.com/github/codeql-action/releases/download/codeql-bundle-20200601/codeql-bundle.tar.gz",
false,
],
];
for (const [
isFeatureFlagEnabled,
toolsInput,
shouldToolcacheBeBypassed,
] of TOOLCACHE_BYPASS_TEST_CASES) {
test(`download codeql bundle ${
shouldToolcacheBeBypassed ? "bypasses" : "does not bypass"
} toolcache when feature flag ${
isFeatureFlagEnabled ? "enabled" : "disabled"
} and tools: ${toolsInput} passed`, async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
await mockApiAndSetupCodeQL({
version: "codeql-bundle-20200601",
apiDetails: sampleApiDetails,
isPinned: true,
tmpDir,
});
t.assert(toolcache.find("CodeQL", "0.0.0-20200601"));
await mockApiAndSetupCodeQL({
version: defaults.bundleVersion,
apiDetails: sampleApiDetails,
featureFlags: createFeatureFlags(
isFeatureFlagEnabled ? [FeatureFlag.BypassToolcacheEnabled] : []
),
toolsInput: { input: toolsInput },
tmpDir,
});
const cachedVersions = toolcache.findAllVersions("CodeQL");
t.is(cachedVersions.length, shouldToolcacheBeBypassed ? 2 : 1);
});
});
}
test("download codeql bundle from github ae endpoint", async (t) => {
await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@ -325,6 +315,7 @@ test("download codeql bundle from github ae endpoint", async (t) => {
sampleGHAEApiDetails,
tmpDir,
util.GitHubVariant.GHAE,
createFeatureFlags([]),
getRunnerLogger(true),
false
);
@ -897,7 +888,7 @@ test(
{}
);
test("does not use injected confg", async (t: ExecutionContext<unknown>) => {
test("does not use injected config", async (t: ExecutionContext<unknown>) => {
const origCODEQL_PASS_CONFIG_TO_CLI = process.env.CODEQL_PASS_CONFIG_TO_CLI;
process.env["CODEQL_PASS_CONFIG_TO_CLI"] = "false";

24
src/codeql.ts
View file

@ -222,7 +222,7 @@ interface PackDownloadItem {
let cachedCodeQL: CodeQL | undefined = undefined;
const CODEQL_BUNDLE_VERSION = defaults.bundleVersion;
const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
export const CODEQL_DEFAULT_ACTION_REPOSITORY = "github/codeql-action";
/**
* The oldest version of CodeQL that the Action will run with. This should be
@ -415,6 +415,7 @@ async function getCodeQLBundleDownloadURL(
* @param apiDetails
* @param tempDir
* @param variant
* @param featureFlags
* @param logger
* @param checkVersion Whether to check that CodeQL CLI meets the minimum
* version requirement. Must be set to true outside tests.
@ -425,14 +426,29 @@ export async function setupCodeQL(
apiDetails: api.GitHubApiDetails,
tempDir: string,
variant: util.GitHubVariant,
featureFlags: FeatureFlags,
logger: Logger,
checkVersion: boolean
): Promise<{ codeql: CodeQL; toolsVersion: string }> {
try {
// We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version.
const forceLatest = codeqlURL === "latest";
const forceLatestReason =
// We use the special value of 'latest' to prioritize the version in the
// defaults over any pinned cached version.
codeqlURL === "latest"
? '"tools: latest" was requested'
: // If the user hasn't requested a particular CodeQL version, then bypass
// the toolcache when the appropriate feature flag is enabled. This
// allows us to quickly rollback a broken bundle that has made its way
// into the toolcache.
codeqlURL === undefined &&
(await featureFlags.getValue(FeatureFlag.BypassToolcacheEnabled))
? "a specific version of CodeQL was not requested and the bypass toolcache feature flag is enabled"
: undefined;
const forceLatest = forceLatestReason !== undefined;
if (forceLatest) {
logger.debug(
`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`
);
codeqlURL = undefined;
}
let codeqlFolder: string;

1
src/config-utils.test.ts
View file

@ -360,6 +360,7 @@ test("load non-empty input", async (t) => {
debugDatabaseName: "my-db",
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
};
const languages = "javascript";

50
src/config-utils.ts
View file

@ -1,5 +1,7 @@
import * as fs from "fs";
import * as path from "path";
// We need to import `performance` on Node 12
import { performance } from "perf_hooks";
import * as yaml from "js-yaml";
import * as semver from "semver";
@ -183,6 +185,11 @@ export interface Config {
* If a key is omitted, then TRAP caching should not be used for that language.
*/
trapCaches: Partial<Record<Language, string>>;
/**
* Time taken to download TRAP caches. Used for status reporting.
*/
trapCacheDownloadTime: number;
}
/**
@ -1023,6 +1030,13 @@ export async function getDefaultConfig(
);
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeQL,
languages,
logger
);
return {
languages,
queries,
@ -1038,12 +1052,30 @@ export async function getDefaultConfig(
debugArtifactName,
debugDatabaseName,
augmentationProperties,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
trapCaches,
trapCacheDownloadTime,
};
}
async function downloadCacheWithTime(
trapCachingEnabled: boolean,
codeQL: CodeQL,
languages: Language[],
logger: Logger
): Promise<{
trapCaches: Partial<Record<Language, string>>;
trapCacheDownloadTime: number;
}> {
let trapCaches = {};
let trapCacheDownloadTime = 0;
if (trapCachingEnabled) {
const start = performance.now();
trapCaches = await downloadTrapCaches(codeQL, languages, logger);
trapCacheDownloadTime = performance.now() - start;
}
return { trapCaches, trapCacheDownloadTime };
}
/**
* Load the config from the given file.
*/
@ -1209,6 +1241,13 @@ async function loadConfig(
}
}
const { trapCaches, trapCacheDownloadTime } = await downloadCacheWithTime(
trapCachingEnabled,
codeQL,
languages,
logger
);
return {
languages,
queries,
@ -1224,9 +1263,8 @@ async function loadConfig(
debugArtifactName,
debugDatabaseName,
augmentationProperties,
trapCaches: trapCachingEnabled
? await downloadTrapCaches(codeQL, languages, logger)
: {},
trapCaches,
trapCacheDownloadTime,
};
}

1
src/database-upload.test.ts
View file

@ -58,6 +58,7 @@ function getTestConfig(tmpDir: string): Config {
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: defaultAugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
};
}

8
src/feature-flags.ts
View file

@ -8,8 +8,9 @@ export interface FeatureFlags {
}
export enum FeatureFlag {
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
BypassToolcacheEnabled = "bypass_toolcache_enabled",
LuaTracerConfigEnabled = "lua_tracer_config_enabled",
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
TrapCachingEnabled = "trap_caching_enabled",
}
@ -32,6 +33,11 @@ export class GitHubFeatureFlags implements FeatureFlags {
) {}
async getValue(flag: FeatureFlag): Promise<boolean> {
// Bypassing the toolcache is disabled in test mode.
if (flag === FeatureFlag.BypassToolcacheEnabled && util.isInTestMode()) {
return false;
}
const response = await this.getApiResponse();
if (response === undefined) {
this.logger.debug(

20
src/init-action.ts
View file

@ -24,8 +24,9 @@ import {
runInit,
} from "./init";
import { Language } from "./languages";
import { getActionsLogger } from "./logging";
import { getActionsLogger, Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import { getTotalCacheSize } from "./trap-caching";
import {
checkActionVersion,
checkGitHubVersionInRange,
@ -65,12 +66,19 @@ interface InitSuccessStatusReport extends StatusReportBase {
tools_resolved_version: string;
/** Comma-separated list of languages specified explicitly in the workflow file. */
workflow_languages: string;
/** Comma-separated list of languages for which we are using TRAP caching. */
trap_cache_languages: string;
/** Size of TRAP caches that we downloaded, in bytes. */
trap_cache_download_size_bytes: number;
/** Time taken to download TRAP caches, in milliseconds. */
trap_cache_download_duration_ms: number;
}
async function sendSuccessStatusReport(
startedAt: Date,
config: configUtils.Config,
toolsVersion: string
toolsVersion: string,
logger: Logger
) {
const statusReportBase = await createStatusReportBase(
"init",
@ -115,6 +123,11 @@ async function sendSuccessStatusReport(
tools_input: getOptionalInput("tools") || "",
tools_resolved_version: toolsVersion,
workflow_languages: workflowLanguages || "",
trap_cache_languages: Object.keys(config.trapCaches).join(","),
trap_cache_download_size_bytes: Math.round(
await getTotalCacheSize(config.trapCaches, logger)
),
trap_cache_download_duration_ms: Math.round(config.trapCacheDownloadTime),
};
await sendStatusReport(statusReport);
@ -172,6 +185,7 @@ async function run() {
apiDetails,
getTemporaryDirectory(),
gitHubVersion.type,
featureFlags,
logger
);
codeql = initCodeQLResult.codeql;
@ -298,7 +312,7 @@ async function run() {
);
return;
}
await sendSuccessStatusReport(startedAt, config, toolsVersion);
await sendSuccessStatusReport(startedAt, config, toolsVersion, logger);
}
async function getTrapCachingEnabled(

2
src/init.ts
View file

@ -20,6 +20,7 @@ export async function initCodeQL(
apiDetails: GitHubApiDetails,
tempDir: string,
variant: util.GitHubVariant,
featureFlags: FeatureFlags,
logger: Logger
): Promise<{ codeql: CodeQL; toolsVersion: string }> {
logger.startGroup("Setup CodeQL tools");
@ -28,6 +29,7 @@ export async function initCodeQL(
apiDetails,
tempDir,
variant,
featureFlags,
logger,
true
);

1
src/runner.ts
View file

@ -229,6 +229,7 @@ program
apiDetails,
tempDir,
gitHubVersion.type,
createFeatureFlags([]),
logger
)
).codeql;

1
src/tracer-config.test.ts
View file

@ -33,6 +33,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: configUtils.defaultAugmentationProperties,
trapCaches: {},
trapCacheDownloadTime: 0,
};
}

2
src/trap-caching.test.ts
View file

@ -87,6 +87,7 @@ const testConfigWithoutTmpDir: Config = {
trapCaches: {
javascript: "/some/cache/dir",
},
trapCacheDownloadTime: 0,
};
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
@ -113,6 +114,7 @@ function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
javascript: path.resolve(tmpDir, "jsCache"),
ruby: path.resolve(tmpDir, "rubyCache"),
},
trapCacheDownloadTime: 0,
};
}

35
src/trap-caching.ts
View file

@ -1,7 +1,9 @@
import * as fs from "fs";
import * as path from "path";
import { promisify } from "util";
import * as cache from "@actions/cache";
import getFolderSize from "get-folder-size";
import * as actionsUtil from "./actions-util";
import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
@ -111,19 +113,26 @@ export async function downloadTrapCaches(
// still just an empty directory. There's no reason to tell the extractor to use it,
// so let's unset the entry in the map so we don't set any extractor options.
logger.info(`No TRAP cache found in Actions cache for ${language}`);
result[language] = undefined;
delete result[language];
}
}
return result;
}
/**
* Possibly upload TRAP caches to the Actions cache.
* @param codeql The CodeQL instance to use.
* @param config The configuration for this workflow.
* @param logger A logger to record some informational messages to.
* @returns Whether the TRAP caches were uploaded.
*/
export async function uploadTrapCaches(
codeql: CodeQL,
config: Config,
logger: Logger
): Promise<void> {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return; // Only upload caches from the default branch
): Promise<boolean> {
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return false; // Only upload caches from the default branch
const toAwait: Array<Promise<number>> = [];
for (const language of config.languages) {
@ -138,6 +147,7 @@ export async function uploadTrapCaches(
toAwait.push(cache.saveCache([cacheDir], key));
}
await Promise.all(toAwait);
return true;
}
export async function getLanguagesSupportingCaching(
@ -175,12 +185,29 @@ export async function getLanguagesSupportingCaching(
return result;
}
export async function getTotalCacheSize(
trapCaches: Partial<Record<Language, string>>,
logger: Logger
): Promise<number> {
try {
const sizes = await Promise.all(
Object.values(trapCaches).map(async (cacheDir) => {
return promisify<string, number>(getFolderSize)(cacheDir);
})
);
return sizes.reduce((a, b) => a + b, 0);
} catch (e) {
logger.warning(`Encountered an error while getting TRAP cache size: ${e}`);
return 0;
}
}
async function cacheKey(
codeql: CodeQL,
language: Language,
baseSha: string
): Promise<string> {
return `${await cachePrefix(codeql, language)}-${baseSha}`;
return `${await cachePrefix(codeql, language)}${baseSha}`;
}
async function cachePrefix(

1
src/util.test.ts
View file

@ -357,6 +357,7 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);

2
src/util.ts
View file

@ -771,7 +771,7 @@ export async function checkActionVersion(version: string) {
* In test mode, we don't upload SARIF results or status reports to the GitHub API.
*/
export function isInTestMode(): boolean {
return process.env["TEST_MODE"] === "true" || false;
return process.env["TEST_MODE"] === "true";
}
/**