Merge remote-tracking branch 'origin/main' into angelapwen/post-init-cleanup
This commit is contained in:
Angela P Wen
commit
90676d9cb9
52 changed files with 985 additions and 69 deletions
|
|
@ -75,7 +75,6 @@ inputs:
|
|||
trap-caching:
|
||||
description: >-
|
||||
Explicitly enable or disable TRAP caching rather than respecting the feature flag for it.
|
||||
This is currently a no-op, as the feature behind this has not been implemented yet.
|
||||
required: false
|
||||
outputs:
|
||||
codeql-path:
|
||||
|
|
|
|||
3
lib/analysis-paths.test.js
generated
3
lib/analysis-paths.test.js
generated
|
|
@ -45,6 +45,7 @@ const util = __importStar(require("./util"));
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
@ -69,6 +70,7 @@ const util = __importStar(require("./util"));
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
|
||||
|
|
@ -93,6 +95,7 @@ const util = __importStar(require("./util"));
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,IAAA,aAAI,EAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;SACzB,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;SACzB,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,EAAE;QACT,iBAAiB,EAAE,EAAE;QACrB,OAAO;QACP,SAAS,EAAE,EAAE;QACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;QACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;QACrD,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,KAAK;QAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,KAAK;KACzB,CAAC;IACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC"}
|
||||
{"version":3,"file":"analysis-paths.test.js","sourceRoot":"","sources":["../src/analysis-paths.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAE7B,8CAAuB;AAEvB,gEAAkD;AAClD,mDAA6C;AAC7C,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,IAAA,aAAI,EAAC,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC7B,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,WAAW,EAAE,EAAE;YACf,KAAK,EAAE,EAAE;YACT,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;YACxB,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;QACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAChC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,MAAM,MAAM,GAAG;YACb,SAAS,EAAE,EAAE;YACb,OAAO,EAAE,EAAE;YACX,KAAK,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YACrC,WAAW,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC;YAC3C,iBAAiB,EAAE,EAAE;YACrB,OAAO,EAAE,MAAM;YACf,SAAS,EAAE,EAAE;YACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;YACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC;YACpD,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,KAAK;YAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;YACnD,iBAAiB,EAAE,KAAK;YACxB,UAAU,EAAE,EAAE;SACf,CAAC;QACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;QACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,EAAE,CACF,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EACjC,gGAAgG,CACjG,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,IAAA,aAAI,EAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,oBAAoB,CAAC,CAAC;IAC/D,MAAM,MAAM,GAAG;QACb,SAAS,EAAE,EAAE;QACb,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,EAAE;QACf,KAAK,EAAE,EAAE;QACT,iBAAiB,EAAE,EAAE;QACrB,OAAO;QACP,SAAS,EAAE,EAAE;QACb,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAwB;QACxE,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC;QACrD,KAAK,EAAE,EAAE;QACT,SAAS,EAAE,KAAK;QAChB,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,IAAI,CAAC,2BAA2B;QACnD,iBAAiB,EAAE,KAAK;QACxB,UAAU,EAAE,EAAE;KACf,CAAC;IACF,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;IACnD,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,oBAAoB,CAAC,CAAC;IAC9D,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,SAAS,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC"}
|
||||
3
lib/analyze-action.js
generated
3
lib/analyze-action.js
generated
|
|
@ -30,6 +30,7 @@ const database_upload_1 = require("./database-upload");
|
|||
const feature_flags_1 = require("./feature-flags");
|
||||
const logging_1 = require("./logging");
|
||||
const repository_1 = require("./repository");
|
||||
const trap_caching_1 = require("./trap-caching");
|
||||
const upload_lib = __importStar(require("./upload-lib"));
|
||||
const util = __importStar(require("./util"));
|
||||
// eslint-disable-next-line import/no-commonjs
|
||||
|
|
@ -97,6 +98,8 @@ async function run() {
|
|||
}
|
||||
// Possibly upload the database bundles for remote queries
|
||||
await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, apiDetails, logger);
|
||||
// Possibly upload the TRAP caches for later re-use
|
||||
await (0, trap_caching_1.uploadTrapCaches)(codeql, config, logger);
|
||||
// We don't upload results in test mode, so don't wait for processing
|
||||
if (util.isInTestMode()) {
|
||||
core.debug("In test mode. Waiting for processing is disabled.");
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
2
lib/analyze.js
generated
2
lib/analyze.js
generated
|
|
@ -79,7 +79,7 @@ async function createdDBForScannedLanguages(codeql, config, logger, featureFlags
|
|||
if (language === languages_1.Language.python) {
|
||||
await setupPythonExtractor(logger);
|
||||
}
|
||||
await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config, language), language, featureFlags);
|
||||
await codeql.extractScannedLanguage(config, language, featureFlags);
|
||||
logger.endGroup();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
2
lib/analyze.test.js
generated
2
lib/analyze.test.js
generated
|
|
@ -119,6 +119,7 @@ const util = __importStar(require("./util"));
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
|
||||
recursive: true,
|
||||
|
|
@ -228,6 +229,7 @@ const stubConfig = {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
for (const options of [
|
||||
{
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
30
lib/codeql.js
generated
30
lib/codeql.js
generated
|
|
@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
|
||||
exports.getExtraOptions = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = exports.CODEQL_VERSION_NEW_TRACING = exports.CODEQL_VERSION_ML_POWERED_QUERIES = exports.CODEQL_VERSION_COUNTS_LINES = exports.CommandInvocationError = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
|
||||
|
|
@ -38,6 +38,7 @@ const error_matcher_1 = require("./error-matcher");
|
|||
const feature_flags_1 = require("./feature-flags");
|
||||
const languages_1 = require("./languages");
|
||||
const toolrunner_error_catcher_1 = require("./toolrunner-error-catcher");
|
||||
const trap_caching_1 = require("./trap-caching");
|
||||
const util = __importStar(require("./util"));
|
||||
const util_1 = require("./util");
|
||||
class CommandInvocationError extends Error {
|
||||
|
|
@ -95,6 +96,11 @@ exports.CODEQL_VERSION_NEW_TRACING = "2.7.0";
|
|||
* some of their files being greater than MAX_PATH (260 characters).
|
||||
*/
|
||||
exports.CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
|
||||
/**
|
||||
* Previous versions had the option already, but were missing the
|
||||
* --extractor-options-verbosity that we need.
|
||||
*/
|
||||
exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
|
||||
function getCodeQLBundleName() {
|
||||
let platform;
|
||||
if (process.platform === "win32") {
|
||||
|
|
@ -356,6 +362,7 @@ function setCodeQL(partialCodeql) {
|
|||
extractScannedLanguage: resolveFunction(partialCodeql, "extractScannedLanguage"),
|
||||
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
|
||||
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
|
||||
betterResolveLanguages: resolveFunction(partialCodeql, "betterResolveLanguages"),
|
||||
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
|
||||
packDownload: resolveFunction(partialCodeql, "packDownload"),
|
||||
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
|
||||
|
|
@ -468,6 +475,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
|
|||
const extraArgs = config.languages.map((language) => `--language=${language}`);
|
||||
if (config.languages.filter(languages_1.isTracedLanguage).length > 0) {
|
||||
extraArgs.push("--begin-tracing");
|
||||
extraArgs.push(...(await (0, trap_caching_1.getTrapCachingExtractorConfigArgs)(config)));
|
||||
if (processName !== undefined) {
|
||||
extraArgs.push(`--trace-process-name=${processName}`);
|
||||
}
|
||||
|
|
@ -525,7 +533,8 @@ async function getCodeQLForCmd(cmd, checkVersion) {
|
|||
// (https://github.com/actions/runner/pull/416).
|
||||
await runTool(autobuildCmd);
|
||||
},
|
||||
async extractScannedLanguage(databasePath, language, featureFlags) {
|
||||
async extractScannedLanguage(config, language, featureFlags) {
|
||||
const databasePath = util.getCodeQLDatabasePath(config, language);
|
||||
// Get extractor location
|
||||
let extractorPath = "";
|
||||
await new toolrunner.ToolRunner(cmd, [
|
||||
|
|
@ -562,6 +571,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
|
|||
"database",
|
||||
"trace-command",
|
||||
...extraArgs,
|
||||
...(await (0, trap_caching_1.getTrapCachingExtractorConfigArgsForLang)(config, language)),
|
||||
...getExtraOptionsFromEnv(["database", "trace-command"]),
|
||||
databasePath,
|
||||
"--",
|
||||
|
|
@ -596,6 +606,22 @@ async function getCodeQLForCmd(cmd, checkVersion) {
|
|||
throw new Error(`Unexpected output from codeql resolve languages: ${e}`);
|
||||
}
|
||||
},
|
||||
async betterResolveLanguages() {
|
||||
const codeqlArgs = [
|
||||
"resolve",
|
||||
"languages",
|
||||
"--format=betterjson",
|
||||
"--extractor-options-verbosity=4",
|
||||
...getExtraOptionsFromEnv(["resolve", "languages"]),
|
||||
];
|
||||
const output = await runTool(cmd, codeqlArgs);
|
||||
try {
|
||||
return JSON.parse(output);
|
||||
}
|
||||
catch (e) {
|
||||
throw new Error(`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`);
|
||||
}
|
||||
},
|
||||
async resolveQueries(queries, extraSearchPath) {
|
||||
const codeqlArgs = [
|
||||
"resolve",
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
lib/codeql.test.js
generated
1
lib/codeql.test.js
generated
|
|
@ -253,6 +253,7 @@ const stubConfig = {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
(0, ava_1.default)("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {
|
||||
const runnerConstructorStub = stubToolRunnerConstructor();
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
17
lib/config-utils.js
generated
17
lib/config-utils.js
generated
|
|
@ -29,6 +29,7 @@ const codeql_1 = require("./codeql");
|
|||
const externalQueries = __importStar(require("./external-queries"));
|
||||
const feature_flags_1 = require("./feature-flags");
|
||||
const languages_1 = require("./languages");
|
||||
const trap_caching_1 = require("./trap-caching");
|
||||
const util_1 = require("./util");
|
||||
// Property names from the user-supplied config file.
|
||||
const NAME_PROPERTY = "name";
|
||||
|
|
@ -462,7 +463,7 @@ function shouldAddConfigFileQueries(queriesInput) {
|
|||
/**
|
||||
* Get the default config for when the user has not supplied one.
|
||||
*/
|
||||
async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
var _a;
|
||||
const languages = await getLanguages(codeQL, languagesInput, repository, apiDetails, logger);
|
||||
const queries = {};
|
||||
|
|
@ -493,13 +494,16 @@ async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLoca
|
|||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
injectedMlQueries,
|
||||
trapCaches: trapCachingEnabled
|
||||
? await (0, trap_caching_1.downloadTrapCaches)(codeQL, languages, logger)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
exports.getDefaultConfig = getDefaultConfig;
|
||||
/**
|
||||
* Load the config from the given file.
|
||||
*/
|
||||
async function loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
async function loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
var _a;
|
||||
let parsedYAML;
|
||||
if (isLocal(configFile)) {
|
||||
|
|
@ -600,6 +604,9 @@ async function loadConfig(languagesInput, queriesInput, packsInput, configFile,
|
|||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
injectedMlQueries,
|
||||
trapCaches: trapCachingEnabled
|
||||
? await (0, trap_caching_1.downloadTrapCaches)(codeQL, languages, logger)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
/**
|
||||
|
|
@ -781,16 +788,16 @@ function dbLocationOrDefault(dbLocation, tempDir) {
|
|||
* This will parse the config from the user input if present, or generate
|
||||
* a default config. The parsed config is then stored to a known location.
|
||||
*/
|
||||
async function initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
async function initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
var _a, _b, _c;
|
||||
let config;
|
||||
// If no config file was provided create an empty one
|
||||
if (!configFile) {
|
||||
logger.debug("No configuration file was provided");
|
||||
config = await getDefaultConfig(languagesInput, queriesInput, packsInput, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
config = await getDefaultConfig(languagesInput, queriesInput, packsInput, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
}
|
||||
else {
|
||||
config = await loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
config = await loadConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
}
|
||||
// The list of queries should not be empty for any language. If it is then
|
||||
// it is a user configuration error.
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
47
lib/config-utils.test.js
generated
47
lib/config-utils.test.js
generated
|
|
@ -88,8 +88,8 @@ function mockListLanguages(languages) {
|
|||
};
|
||||
},
|
||||
});
|
||||
const config = await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
t.deepEqual(config, await configUtils.getDefaultConfig(languages, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger));
|
||||
const config = await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
t.deepEqual(config, await configUtils.getDefaultConfig(languages, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger));
|
||||
});
|
||||
});
|
||||
(0, ava_1.default)("loading config saves config", async (t) => {
|
||||
|
|
@ -111,7 +111,7 @@ function mockListLanguages(languages) {
|
|||
t.false(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
|
||||
// Sanity check that getConfig returns undefined before we have called initConfig
|
||||
t.deepEqual(await configUtils.getConfig(tmpDir, logger), undefined);
|
||||
const config1 = await configUtils.initConfig("javascript,python", undefined, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
const config1 = await configUtils.initConfig("javascript,python", undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
// The saved config file should now exist
|
||||
t.true(fs.existsSync(configUtils.getPathToParsedConfigFile(tmpDir)));
|
||||
// And that same newly-initialised config should now be returned by getConfig
|
||||
|
|
@ -125,7 +125,7 @@ function mockListLanguages(languages) {
|
|||
(0, ava_1.default)("load input outside of workspace", async (t) => {
|
||||
return await util.withTmpDir(async (tmpDir) => {
|
||||
try {
|
||||
await configUtils.initConfig(undefined, undefined, undefined, "../input", undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(undefined, undefined, undefined, "../input", undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -138,7 +138,7 @@ function mockListLanguages(languages) {
|
|||
// no filename given, just a repo
|
||||
const configFile = "octo-org/codeql-config@main";
|
||||
try {
|
||||
await configUtils.initConfig(undefined, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(undefined, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -152,7 +152,7 @@ function mockListLanguages(languages) {
|
|||
const configFile = "input";
|
||||
t.false(fs.existsSync(path.join(tmpDir, configFile)));
|
||||
try {
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -220,10 +220,11 @@ function mockListLanguages(languages) {
|
|||
debugArtifactName: "my-artifact",
|
||||
debugDatabaseName: "my-db",
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
const languages = "javascript";
|
||||
const configFilePath = createConfigFile(inputFileContents, tmpDir);
|
||||
const actualConfig = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, "my-artifact", "my-db", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const actualConfig = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, false, "my-artifact", "my-db", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Should exactly equal the object we constructed earlier
|
||||
t.deepEqual(actualConfig, expectedConfig);
|
||||
});
|
||||
|
|
@ -259,7 +260,7 @@ function mockListLanguages(languages) {
|
|||
fs.mkdirSync(path.join(tmpDir, "foo"));
|
||||
const languages = "javascript";
|
||||
const configFilePath = createConfigFile(inputFileContents, tmpDir);
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolve queries was called correctly
|
||||
t.deepEqual(resolveQueriesArgs.length, 1);
|
||||
t.deepEqual(resolveQueriesArgs[0].queries, [
|
||||
|
|
@ -302,7 +303,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
const languages = "javascript";
|
||||
const config = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const config = await configUtils.initConfig(languages, undefined, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries
|
||||
// and once for `./foo` from the config file.
|
||||
|
|
@ -335,7 +336,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
const languages = "javascript";
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries and once for `./override`,
|
||||
// but won't be called for './foo' from the config file.
|
||||
|
|
@ -367,7 +368,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
const languages = "javascript";
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for `./workflow-query`,
|
||||
// but won't be called for the default one since that was disabled
|
||||
|
|
@ -393,7 +394,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
const languages = "javascript";
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolveQueries was called correctly:
|
||||
// It'll be called once for the default queries,
|
||||
// and then once for each of the two queries from the workflow
|
||||
|
|
@ -432,7 +433,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
const languages = "javascript";
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const config = await configUtils.initConfig(languages, testQueries, undefined, configFilePath, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
// Check resolveQueries was called correctly
|
||||
// It'll be called once for the default queries,
|
||||
// once for each of additional1 and additional2,
|
||||
|
|
@ -471,7 +472,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
try {
|
||||
await configUtils.initConfig(languages, queries, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, queries, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
t.fail("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -514,7 +515,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
fs.mkdirSync(path.join(tmpDir, "foo/bar/dev"), { recursive: true });
|
||||
const configFile = "octo-org/codeql-config/config.yaml@main";
|
||||
const languages = "javascript";
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
t.assert(spyGetContents.called);
|
||||
});
|
||||
});
|
||||
|
|
@ -524,7 +525,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
mockGetContents(dummyResponse);
|
||||
const repoReference = "octo-org/codeql-config/config.yaml@main";
|
||||
try {
|
||||
await configUtils.initConfig(undefined, undefined, undefined, repoReference, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(undefined, undefined, undefined, repoReference, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -540,7 +541,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
mockGetContents(dummyResponse);
|
||||
const repoReference = "octo-org/codeql-config/config.yaml@main";
|
||||
try {
|
||||
await configUtils.initConfig(undefined, undefined, undefined, repoReference, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(undefined, undefined, undefined, repoReference, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -557,7 +558,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
},
|
||||
});
|
||||
try {
|
||||
await configUtils.initConfig(undefined, undefined, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(undefined, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -569,7 +570,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
return await util.withTmpDir(async (tmpDir) => {
|
||||
const languages = "rubbish,english";
|
||||
try {
|
||||
await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, undefined, undefined, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, (0, codeql_1.getCachedCodeQL)(), tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -597,7 +598,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
const configFile = path.join(tmpDir, "codeql-config.yaml");
|
||||
fs.writeFileSync(configFile, inputFileContents);
|
||||
const languages = "javascript";
|
||||
const { packs } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const { packs } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
t.deepEqual(packs, {
|
||||
[languages_1.Language.javascript]: ["a/b@1.2.3"],
|
||||
});
|
||||
|
|
@ -631,7 +632,7 @@ function queriesToResolvedQueryForm(queries) {
|
|||
fs.writeFileSync(configFile, inputFileContents);
|
||||
fs.mkdirSync(path.join(tmpDir, "foo"));
|
||||
const languages = "javascript,python,cpp";
|
||||
const { packs, queries } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example" }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
const { packs, queries } = await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example" }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
t.deepEqual(packs, {
|
||||
[languages_1.Language.javascript]: ["a/b@1.2.3"],
|
||||
[languages_1.Language.python]: ["c/d@1.2.3"],
|
||||
|
|
@ -674,7 +675,7 @@ function doInvalidInputTest(testName, inputFileContents, expectedErrorMessageGen
|
|||
const inputFile = path.join(tmpDir, configFile);
|
||||
fs.writeFileSync(inputFile, inputFileContents, "utf8");
|
||||
try {
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
await configUtils.initConfig(languages, undefined, undefined, configFile, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)([]), (0, logging_1.getRunnerLogger)(true));
|
||||
throw new Error("initConfig did not throw error");
|
||||
}
|
||||
catch (err) {
|
||||
|
|
@ -891,7 +892,7 @@ const mlPoweredQueriesMacro = ava_1.default.macro({
|
|||
};
|
||||
},
|
||||
});
|
||||
const { packs } = await configUtils.initConfig("javascript", queriesInput, packsInput, undefined, undefined, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
|
||||
const { packs } = await configUtils.initConfig("javascript", queriesInput, packsInput, undefined, undefined, false, false, "", "", { owner: "github", repo: "example " }, tmpDir, codeQL, tmpDir, gitHubVersion, sampleApiDetails, (0, feature_flags_1.createFeatureFlags)(isMlPoweredQueriesFlagEnabled
|
||||
? [feature_flags_1.FeatureFlag.MlPoweredQueriesEnabled]
|
||||
: []), (0, logging_1.getRunnerLogger)(true));
|
||||
if (expectedVersionString !== undefined) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
lib/database-upload.test.js
generated
1
lib/database-upload.test.js
generated
|
|
@ -58,6 +58,7 @@ function getTestConfig(tmpDir) {
|
|||
debugArtifactName: util_1.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util_1.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
}
|
||||
async function mockHttpRequests(databaseUploadStatusCode) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
lib/feature-flags.js
generated
1
lib/feature-flags.js
generated
|
|
@ -26,6 +26,7 @@ var FeatureFlag;
|
|||
(function (FeatureFlag) {
|
||||
FeatureFlag["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
|
||||
FeatureFlag["LuaTracerConfigEnabled"] = "lua_tracer_config_enabled";
|
||||
FeatureFlag["TrapCachingEnabled"] = "trap_caching_enabled";
|
||||
})(FeatureFlag = exports.FeatureFlag || (exports.FeatureFlag = {}));
|
||||
class GitHubFeatureFlags {
|
||||
constructor(gitHubVersion, apiDetails, repositoryNwo, logger) {
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAG9D,6CAA+B;AAM/B,IAAY,WAGX;AAHD,WAAY,WAAW;IACrB,qEAAsD,CAAA;IACtD,mEAAoD,CAAA;AACtD,CAAC,EAHW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAGtB;AAUD,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,aAA4B,EAC5B,MAAc;QAHd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,IAAiB;QAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qCAAqC,IAAI,4BAA4B,CACtE,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;oBAC/B,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;iBAC9B,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;oBAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gGAAgG;wBAC9F,oEAAoE;wBACpE,qFAAqF;wBACrF,kFAAkF,CAAC,EAAE,CACxF,CAAC;iBACH;qBAAM;oBACL,uFAAuF;oBACvF,mFAAmF;oBACnF,2FAA2F;oBAC3F,qBAAqB;oBACrB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAAC,EAAE,CAChE,CAAC;iBACH;aACF;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAvED,gDAuEC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAAC,YAA2B;IAC5D,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;AACJ,CAAC;AAND,gDAMC"}
|
||||
{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAG9D,6CAA+B;AAM/B,IAAY,WAIX;AAJD,WAAY,WAAW;IACrB,qEAAsD,CAAA;IACtD,mEAAoD,CAAA;IACpD,0DAA2C,CAAA;AAC7C,CAAC,EAJW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAItB;AAUD,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,aAA4B,EAC5B,MAAc;QAHd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,IAAiB;QAC9B,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qCAAqC,IAAI,4BAA4B,CACtE,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;oBAC/B,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;iBAC9B,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;oBAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gGAAgG;wBAC9F,oEAAoE;wBACpE,qFAAqF;wBACrF,kFAAkF,CAAC,EAAE,CACxF,CAAC;iBACH;qBAAM;oBACL,uFAAuF;oBACvF,mFAAmF;oBACnF,2FAA2F;oBAC3F,qBAAqB;oBACrB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAAC,EAAE,CAChE,CAAC;iBACH;aACF;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAvED,gDAuEC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAAC,YAA2B;IAC5D,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;AACJ,CAAC;AAND,gDAMC"}
|
||||
8
lib/init-action.js
generated
8
lib/init-action.js
generated
|
|
@ -93,7 +93,7 @@ async function run() {
|
|||
codeql = initCodeQLResult.codeql;
|
||||
toolsVersion = initCodeQLResult.toolsVersion;
|
||||
await (0, util_1.enrichEnvironment)(util_1.Mode.actions, codeql);
|
||||
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"),
|
||||
config = await (0, init_1.initConfig)((0, actions_util_1.getOptionalInput)("languages"), (0, actions_util_1.getOptionalInput)("queries"), (0, actions_util_1.getOptionalInput)("packs"), (0, actions_util_1.getOptionalInput)("config-file"), (0, actions_util_1.getOptionalInput)("db-location"), await getTrapCachingEnabled(featureFlags),
|
||||
// Debug mode is enabled if:
|
||||
// - The `init` Action is passed `debug: true`.
|
||||
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
|
||||
|
|
@ -153,6 +153,12 @@ async function run() {
|
|||
}
|
||||
await sendSuccessStatusReport(startedAt, config, toolsVersion);
|
||||
}
|
||||
async function getTrapCachingEnabled(featureFlags) {
|
||||
const trapCaching = (0, actions_util_1.getOptionalInput)("trap-caching");
|
||||
if (trapCaching !== undefined)
|
||||
return trapCaching === "true";
|
||||
return await featureFlags.getValue(feature_flags_1.FeatureFlag.TrapCachingEnabled);
|
||||
}
|
||||
async function runWrapper() {
|
||||
try {
|
||||
await run();
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
4
lib/init.js
generated
4
lib/init.js
generated
|
|
@ -38,9 +38,9 @@ async function initCodeQL(codeqlURL, apiDetails, tempDir, variant, logger) {
|
|||
return { codeql, toolsVersion };
|
||||
}
|
||||
exports.initCodeQL = initCodeQL;
|
||||
async function initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
async function initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger) {
|
||||
logger.startGroup("Load language configuration");
|
||||
const config = await configUtils.initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
const config = await configUtils.initConfig(languagesInput, queriesInput, packsInput, configFile, dbLocation, trapCachingEnabled, debugMode, debugArtifactName, debugDatabaseName, repository, tempDir, codeQL, workspacePath, gitHubVersion, apiDetails, featureFlags, logger);
|
||||
analysisPaths.printPathFiltersWarning(config, logger);
|
||||
logger.endGroup();
|
||||
return config;
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,gEAAkD;AAElD,qCAA2E;AAC3E,4DAA8C;AAI9C,mDAAwE;AACxE,6CAA+B;AAC/B,iCAA4C;AAErC,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,IAAA,oBAAW,EAChD,SAAS,EACT,UAAU,EACV,OAAO,EACP,OAAO,EACP,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AAClC,CAAC;AAnBD,gCAmBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAA8B,EAC9B,UAA8B,EAC9B,SAAkB,EAClB,iBAAyB,EACzB,iBAAyB,EACzB,UAAyB,EACzB,OAAe,EACf,MAAc,EACd,aAAqB,EACrB,aAAiC,EACjC,UAAoC,EACpC,YAA0B,EAC1B,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACT,iBAAiB,EACjB,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,MAAM,EACN,aAAa,EACb,aAAa,EACb,UAAU,EACV,YAAY,EACZ,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AAxCD,gCAwCC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,YAAgC,EAChC,YAA0B;;IAE1B,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,IAAI;QACF,IAAI,MAAM,IAAA,yBAAkB,EAAC,MAAM,EAAE,mCAA0B,CAAC,EAAE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,YAAY,CACb,CAAC;SACH;aAAM;YACL,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,yBAAyB;gBACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,EAC5C,QAAQ,EACR,UAAU,CACX,CAAC;aACH;SACF;KACF;IAAC,OAAO,CAAC,EAAE;QACV,kDAAkD;QAClD,yCAAyC;QACzC,IACE,CAAC,YAAY,KAAK;aAClB,MAAA,CAAC,CAAC,OAAO,0CAAE,QAAQ,CAAC,8BAA8B,CAAC,CAAA;YACnD,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,uCAAuC,CAAC,EAC3D;YACA,MAAM,IAAI,IAAI,CAAC,SAAS,CACtB,sDAAsD,CAAC,CAAC,OAAO,EAAE,CAClE,CAAC;SACH;aAAM,IACL,CAAC,YAAY,KAAK;aAClB,MAAA,CAAC,CAAC,OAAO,0CAAE,QAAQ,CAAC,wCAAwC,CAAC,CAAA,EAC7D;YACA,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;IACD,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnDD,0BAmDC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;;;;;gDAqBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EACvC;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AA5FD,kDA4FC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,IAAI;QACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;gBACvE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;aAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QACD,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBAC/D,IAAI;gBACJ,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE;gBACpE,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;KACF;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,gFAAgF,CAAC,IAAI;YACnF,qGAAqG;YACrG,oGAAoG;YACpG,iDAAiD,CACpD,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAzCD,8CAyCC"}
|
||||
{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,yEAA2D;AAC3D,kEAAoD;AAEpD,gEAAkD;AAElD,qCAA2E;AAC3E,4DAA8C;AAI9C,mDAAwE;AACxE,6CAA+B;AAC/B,iCAA4C;AAErC,KAAK,UAAU,UAAU,CAC9B,SAA6B,EAC7B,UAA4B,EAC5B,OAAe,EACf,OAA2B,EAC3B,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,IAAA,oBAAW,EAChD,SAAS,EACT,UAAU,EACV,OAAO,EACP,OAAO,EACP,MAAM,EACN,IAAI,CACL,CAAC;IACF,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;IAC5B,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;AAClC,CAAC;AAnBD,gCAmBC;AAEM,KAAK,UAAU,UAAU,CAC9B,cAAkC,EAClC,YAAgC,EAChC,UAA8B,EAC9B,UAA8B,EAC9B,UAA8B,EAC9B,kBAA2B,EAC3B,SAAkB,EAClB,iBAAyB,EACzB,iBAAyB,EACzB,UAAyB,EACzB,OAAe,EACf,MAAc,EACd,aAAqB,EACrB,aAAiC,EACjC,UAAoC,EACpC,YAA0B,EAC1B,MAAc;IAEd,MAAM,CAAC,UAAU,CAAC,6BAA6B,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CACzC,cAAc,EACd,YAAY,EACZ,UAAU,EACV,UAAU,EACV,UAAU,EACV,kBAAkB,EAClB,SAAS,EACT,iBAAiB,EACjB,iBAAiB,EACjB,UAAU,EACV,OAAO,EACP,MAAM,EACN,aAAa,EACb,aAAa,EACb,UAAU,EACV,YAAY,EACZ,MAAM,CACP,CAAC;IACF,aAAa,CAAC,uBAAuB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtD,MAAM,CAAC,QAAQ,EAAE,CAAC;IAClB,OAAO,MAAM,CAAC;AAChB,CAAC;AA1CD,gCA0CC;AAEM,KAAK,UAAU,OAAO,CAC3B,MAAc,EACd,MAA0B,EAC1B,UAAkB,EAClB,WAA+B,EAC/B,YAAgC,EAChC,YAA0B;;IAE1B,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAErD,IAAI;QACF,IAAI,MAAM,IAAA,yBAAkB,EAAC,MAAM,EAAE,mCAA0B,CAAC,EAAE;YAChE,0BAA0B;YAC1B,MAAM,MAAM,CAAC,mBAAmB,CAC9B,MAAM,EACN,UAAU,EACV,WAAW,EACX,YAAY,EACZ,YAAY,CACb,CAAC;SACH;aAAM;YACL,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;gBACvC,yBAAyB;gBACzB,MAAM,MAAM,CAAC,YAAY,CACvB,IAAI,CAAC,qBAAqB,CAAC,MAAM,EAAE,QAAQ,CAAC,EAC5C,QAAQ,EACR,UAAU,CACX,CAAC;aACH;SACF;KACF;IAAC,OAAO,CAAC,EAAE;QACV,kDAAkD;QAClD,yCAAyC;QACzC,IACE,CAAC,YAAY,KAAK;aAClB,MAAA,CAAC,CAAC,OAAO,0CAAE,QAAQ,CAAC,8BAA8B,CAAC,CAAA;YACnD,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,uCAAuC,CAAC,EAC3D;YACA,MAAM,IAAI,IAAI,CAAC,SAAS,CACtB,sDAAsD,CAAC,CAAC,OAAO,EAAE,CAClE,CAAC;SACH;aAAM,IACL,CAAC,YAAY,KAAK;aAClB,MAAA,CAAC,CAAC,OAAO,0CAAE,QAAQ,CAAC,wCAAwC,CAAC,CAAA,EAC7D;YACA,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACrC;aAAM;YACL,MAAM,CAAC,CAAC;SACT;KACF;IACD,OAAO,MAAM,IAAA,uCAAuB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACvD,CAAC;AAnDD,0BAmDC;AAED,sEAAsE;AACtE,4EAA4E;AAC5E,4EAA4E;AAC5E,6EAA6E;AAC7E,+CAA+C;AACxC,KAAK,UAAU,mBAAmB,CACvC,WAA+B,EAC/B,YAAgC,EAChC,MAA0B,EAC1B,MAAc,EACd,YAA0B;IAE1B,IAAI,MAAc,CAAC;IACnB,IAAI,WAAW,KAAK,SAAS,EAAE;QAC7B,MAAM,GAAG;;;;;;;;;;;;uCAY0B,WAAW;;8BAEpB,WAAW;;;;;;;;gDAQO,CAAC;KAC9C;SAAM;QACL,oEAAoE;QACpE,mFAAmF;QACnF,+EAA+E;QAC/E,kFAAkF;QAClF,6EAA6E;QAC7E,oFAAoF;QACpF,6CAA6C;QAC7C,YAAY,GAAG,YAAY,IAAI,CAAC,CAAC;QACjC,MAAM,GAAG;;;;;;;;4BAQe,YAAY;;;;;;;;;;;;;;;;;;;;;gDAqBQ,CAAC;KAC9C;IAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACxE,EAAE,CAAC,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAE3C,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EACvC;QACE,kBAAkB;QAClB,QAAQ;QACR,OAAO;QACP,gBAAgB;QAChB,IAAI,CAAC,OAAO,CACV,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAC9B,OAAO,EACP,OAAO,EACP,YAAY,CACb;KACF,EACD,EAAE,GAAG,EAAE,EAAE,0BAA0B,EAAE,YAAY,CAAC,IAAI,EAAE,EAAE,CAC3D,CAAC,IAAI,EAAE,CAAC;AACX,CAAC;AA5FD,kDA4FC;AAEM,KAAK,UAAU,iBAAiB,CAAC,MAAc,EAAE,MAAc;IACpE,MAAM,CAAC,UAAU,CAAC,2BAA2B,CAAC,CAAC;IAE/C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC;IAEjE,IAAI;QACF,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE;gBACvE,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,mBAAmB,CAAC;aAC9C,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,UAAU,CAAC,UAAU,CAC7B,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,kBAAkB,CAAC,CAC7C,CAAC,IAAI,EAAE,CAAC;SACV;QACD,MAAM,MAAM,GAAG,0BAA0B,CAAC;QAC1C,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;YAChC,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE;gBAC/D,IAAI;gBACJ,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;aAAM;YACL,MAAM,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,SAAS,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE;gBACpE,IAAI;gBACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC/B,CAAC,CAAC,IAAI,EAAE,CAAC;SACX;KACF;IAAC,OAAO,CAAC,EAAE;QACV,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,CAAC,OAAO,CACZ,gFAAgF,CAAC,IAAI;YACnF,qGAAqG;YACrG,oGAAoG;YACpG,iDAAiD,CACpD,CAAC;QACF,OAAO;KACR;IACD,MAAM,CAAC,QAAQ,EAAE,CAAC;AACpB,CAAC;AAzCD,8CAyCC"}
|
||||
2
lib/runner.js
generated
2
lib/runner.js
generated
|
|
@ -155,7 +155,7 @@ program
|
|||
}
|
||||
await (0, util_1.enrichEnvironment)(util_1.Mode.runner, codeql);
|
||||
const workspacePath = checkoutPath;
|
||||
const config = await (0, init_1.initConfig)(cmd.languages, cmd.queries, cmd.packs, cmd.configFile, undefined, false, "", "", (0, repository_1.parseRepositoryNwo)(cmd.repository), tempDir, codeql, workspacePath, gitHubVersion, apiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
const config = await (0, init_1.initConfig)(cmd.languages, cmd.queries, cmd.packs, cmd.configFile, undefined, false, false, "", "", (0, repository_1.parseRepositoryNwo)(cmd.repository), tempDir, codeql, workspacePath, gitHubVersion, apiDetails, (0, feature_flags_1.createFeatureFlags)([]), logger);
|
||||
const sourceRoot = checkoutPath;
|
||||
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, parseTraceProcessName(), parseTraceProcessLevel(), (0, feature_flags_1.createFeatureFlags)([]));
|
||||
if (tracerConfig === undefined) {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
1
lib/tracer-config.test.js
generated
1
lib/tracer-config.test.js
generated
|
|
@ -47,6 +47,7 @@ function getTestConfig(tmpDir) {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
}
|
||||
// A very minimal setup
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
156
lib/trap-caching.js
generated
Normal file
156
lib/trap-caching.js
generated
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.getLanguagesSupportingCaching = exports.uploadTrapCaches = exports.downloadTrapCaches = exports.getTrapCachingExtractorConfigArgsForLang = exports.getTrapCachingExtractorConfigArgs = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const cache = __importStar(require("@actions/cache"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const util_1 = require("./util");
|
||||
// This constant should be bumped if we make a breaking change
|
||||
// to how the CodeQL Action stores or retrieves the TRAP cache,
|
||||
// and will invalidate previous caches. We don't need to bump
|
||||
// this for CLI/extractor changes, since the CLI version also
|
||||
// goes into the cache key.
|
||||
const CACHE_VERSION = 1;
|
||||
// This constant sets the size of each TRAP cache in megabytes.
|
||||
const CACHE_SIZE_MB = 1024;
|
||||
async function getTrapCachingExtractorConfigArgs(config) {
|
||||
const result = [];
|
||||
for (const language of config.languages)
|
||||
result.push(await getTrapCachingExtractorConfigArgsForLang(config, language));
|
||||
return result.flat();
|
||||
}
|
||||
exports.getTrapCachingExtractorConfigArgs = getTrapCachingExtractorConfigArgs;
|
||||
async function getTrapCachingExtractorConfigArgsForLang(config, language) {
|
||||
const cacheDir = config.trapCaches[language];
|
||||
if (cacheDir === undefined)
|
||||
return [];
|
||||
const write = await actionsUtil.isAnalyzingDefaultBranch();
|
||||
return [
|
||||
`-O=${language}.trap.cache.dir=${cacheDir}`,
|
||||
`-O=${language}.trap.cache.bound=${CACHE_SIZE_MB}`,
|
||||
`-O=${language}.trap.cache.write=${write}`,
|
||||
];
|
||||
}
|
||||
exports.getTrapCachingExtractorConfigArgsForLang = getTrapCachingExtractorConfigArgsForLang;
|
||||
/**
|
||||
* Download TRAP caches from the Actions cache.
|
||||
* @param codeql The CodeQL instance to use.
|
||||
* @param languages The languages being analyzed.
|
||||
* @param logger A logger to record some informational messages to.
|
||||
* @returns A partial map from languages to TRAP cache paths on disk, with
|
||||
* languages for which we shouldn't use TRAP caching omitted.
|
||||
*/
|
||||
async function downloadTrapCaches(codeql, languages, logger) {
|
||||
var _a, _b;
|
||||
const result = {};
|
||||
const languagesSupportingCaching = await getLanguagesSupportingCaching(codeql, languages, logger);
|
||||
logger.info(`Found ${languagesSupportingCaching.length} languages that support TRAP caching`);
|
||||
if (languagesSupportingCaching.length === 0)
|
||||
return result;
|
||||
const cachesDir = path.join(actionsUtil.getTemporaryDirectory(), "trapCaches");
|
||||
for (const language of languagesSupportingCaching) {
|
||||
const cacheDir = path.join(cachesDir, language);
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
result[language] = cacheDir;
|
||||
}
|
||||
if (await actionsUtil.isAnalyzingDefaultBranch()) {
|
||||
logger.info("Analyzing default branch. Skipping downloading of TRAP caches.");
|
||||
return result;
|
||||
}
|
||||
let baseSha = "unknown";
|
||||
const eventPath = process.env.GITHUB_EVENT_PATH;
|
||||
if (process.env.GITHUB_EVENT_NAME === "pull_request" &&
|
||||
eventPath !== undefined) {
|
||||
const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8"));
|
||||
baseSha = ((_b = (_a = event.pull_request) === null || _a === void 0 ? void 0 : _a.base) === null || _b === void 0 ? void 0 : _b.sha) || baseSha;
|
||||
}
|
||||
for (const language of languages) {
|
||||
const cacheDir = result[language];
|
||||
if (cacheDir === undefined)
|
||||
continue;
|
||||
// The SHA from the base of the PR is the most similar commit we might have a cache for
|
||||
const preferredKey = await cacheKey(codeql, language, baseSha);
|
||||
logger.info(`Looking in Actions cache for TRAP cache with key ${preferredKey}`);
|
||||
const found = await cache.restoreCache([cacheDir], preferredKey, [
|
||||
await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
|
||||
]);
|
||||
if (found === undefined) {
|
||||
// We didn't find a TRAP cache in the Actions cache, so the directory on disk is
|
||||
// still just an empty directory. There's no reason to tell the extractor to use it,
|
||||
// so let's unset the entry in the map so we don't set any extractor options.
|
||||
logger.info(`No TRAP cache found in Actions cache for ${language}`);
|
||||
result[language] = undefined;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
exports.downloadTrapCaches = downloadTrapCaches;
|
||||
async function uploadTrapCaches(codeql, config, logger) {
|
||||
if (!(await actionsUtil.isAnalyzingDefaultBranch()))
|
||||
return; // Only upload caches from the default branch
|
||||
const toAwait = [];
|
||||
for (const language of config.languages) {
|
||||
const cacheDir = config.trapCaches[language];
|
||||
if (cacheDir === undefined)
|
||||
continue;
|
||||
const key = await cacheKey(codeql, language, process.env.GITHUB_SHA || "unknown");
|
||||
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
|
||||
toAwait.push(cache.saveCache([cacheDir], key));
|
||||
}
|
||||
await Promise.all(toAwait);
|
||||
}
|
||||
exports.uploadTrapCaches = uploadTrapCaches;
|
||||
async function getLanguagesSupportingCaching(codeql, languages, logger) {
|
||||
var _a, _b, _c, _d;
|
||||
const result = [];
|
||||
if (!(await (0, util_1.codeQlVersionAbove)(codeql, codeql_1.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES)))
|
||||
return result;
|
||||
const resolveResult = await codeql.betterResolveLanguages();
|
||||
outer: for (const lang of languages) {
|
||||
if (resolveResult.extractors[lang].length !== 1)
|
||||
continue;
|
||||
const extractor = resolveResult.extractors[lang][0];
|
||||
const trapCacheOptions = (_d = (_c = (_b = (_a = extractor.extractor_options) === null || _a === void 0 ? void 0 : _a.trap) === null || _b === void 0 ? void 0 : _b.properties) === null || _c === void 0 ? void 0 : _c.cache) === null || _d === void 0 ? void 0 : _d.properties;
|
||||
if (trapCacheOptions === undefined) {
|
||||
logger.info(`${lang} does not support TRAP caching (missing option group)`);
|
||||
continue;
|
||||
}
|
||||
for (const requiredOpt of ["dir", "bound", "write"]) {
|
||||
if (!(requiredOpt in trapCacheOptions)) {
|
||||
logger.info(`${lang} does not support TRAP caching (missing ${requiredOpt} option)`);
|
||||
continue outer;
|
||||
}
|
||||
}
|
||||
result.push(lang);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
exports.getLanguagesSupportingCaching = getLanguagesSupportingCaching;
|
||||
async function cacheKey(codeql, language, baseSha) {
|
||||
return `${await cachePrefix(codeql, language)}-${baseSha}`;
|
||||
}
|
||||
async function cachePrefix(codeql, language) {
|
||||
return `codeql-trap-${CACHE_VERSION}-${await codeql.getVersion()}-${language}-`;
|
||||
}
|
||||
//# sourceMappingURL=trap-caching.js.map
|
||||
1
lib/trap-caching.js.map
Normal file
1
lib/trap-caching.js.map
Normal file
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"trap-caching.js","sourceRoot":"","sources":["../src/trap-caching.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,sDAAwC;AAExC,4DAA8C;AAC9C,qCAA2E;AAI3E,iCAA4C;AAE5C,8DAA8D;AAC9D,+DAA+D;AAC/D,6DAA6D;AAC7D,6DAA6D;AAC7D,2BAA2B;AAC3B,MAAM,aAAa,GAAG,CAAC,CAAC;AAExB,+DAA+D;AAC/D,MAAM,aAAa,GAAG,IAAI,CAAC;AAEpB,KAAK,UAAU,iCAAiC,CACrD,MAAc;IAEd,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS;QACrC,MAAM,CAAC,IAAI,CACT,MAAM,wCAAwC,CAAC,MAAM,EAAE,QAAQ,CAAC,CACjE,CAAC;IACJ,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;AACvB,CAAC;AATD,8EASC;AAEM,KAAK,UAAU,wCAAwC,CAC5D,MAAc,EACd,QAAkB;IAElB,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC7C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACtC,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,wBAAwB,EAAE,CAAC;IAC3D,OAAO;QACL,MAAM,QAAQ,mBAAmB,QAAQ,EAAE;QAC3C,MAAM,QAAQ,qBAAqB,aAAa,EAAE;QAClD,MAAM,QAAQ,qBAAqB,KAAK,EAAE;KAC3C,CAAC;AACJ,CAAC;AAZD,4FAYC;AAED;;;;;;;GAOG;AACI,KAAK,UAAU,kBAAkB,CACtC,MAAc,EACd,SAAqB,EACrB,MAAc;;IAEd,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,MAAM,0BAA0B,GAAG,MAAM,6BAA6B,CACpE,MAAM,EACN,SAAS,EACT,MAAM,CACP,CAAC;IACF,MAAM,CAAC,IAAI,CACT,SAAS,0BAA0B,CAAC,MAAM,sCAAsC,CACjF,CAAC;IACF,IAAI,0BAA0B,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAE3D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CACzB,WAAW,CAAC,qBAAqB,EAAE,EACnC,YAAY,CACb,CAAC;IACF,KAAK,MAAM,QAAQ,IAAI,0BAA0B,EAAE;QACjD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAChD,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,CAAC,QAAQ,CAAC,GAAG,QAAQ,CAAC;KAC7B;IAED,IAAI,MAAM,WAAW,CAAC,wBAAwB,EAAE,EAAE;QAChD,MAAM,CAAC,IAAI,CACT,gEAAgE,CACjE,CAAC;QACF,OAAO,MAAM,CAAC;KACf;IAED,IAAI,OAAO,GAAG,SAAS,CAAC;IACxB,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;IAChD,IACE,OAAO,CAAC,GAAG,CAAC,iBAAiB,KAAK,cAAc;QAChD,SAAS,KAAK,SAAS,EACvB;QACA,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QAC5E,OAAO,GAAG,CAAA,MAAA,MAAA,KAAK,CAAC,YAAY,0CAAE,IAAI,0CAAE,GAAG,KAAI,OAAO,CAAC;KACpD;IACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;QAClC,IAAI,QAAQ,KAAK,SAAS;YAAE,SAAS;QACrC,uFAAuF;QACvF,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,CAAC,IAAI,CACT,oDAAoD,YAAY,EAAE,CACnE,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,EAAE,YAAY,EAAE;YAC/D,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,mDAAmD;SACzF,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,SAAS,EAAE;YACvB,gFAAgF;YAChF,oFAAoF;YACpF,6EAA6E;YAC7E,MAAM,CAAC,IAAI,CAAC,4CAA4C,QAAQ,EAAE,CAAC,CAAC;YACpE,MAAM,CAAC,QAAQ,CAAC,GAAG,SAAS,CAAC;SAC9B;KACF;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AA/DD,gDA+DC;AAEM,KAAK,UAAU,gBAAgB,CACpC,MAAc,EACd,MAAc,EACd,MAAc;IAEd,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,wBAAwB,EAAE,CAAC;QAAE,OAAO,CAAC,6CAA6C;IAE1G,MAAM,OAAO,GAA2B,EAAE,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS;YAAE,SAAS;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CACxB,MAAM,EACN,QAAQ,EACR,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,SAAS,CACpC,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,kDAAkD,GAAG,EAAE,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;KAChD;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AApBD,4CAoBC;AAEM,KAAK,UAAU,6BAA6B,CACjD,MAAc,EACd,SAAqB,EACrB,MAAc;;IAEd,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IACE,CAAC,CAAC,MAAM,IAAA,yBAAkB,EAAC,MAAM,EAAE,gDAAuC,CAAC,CAAC;QAE5E,OAAO,MAAM,CAAC;IAChB,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,sBAAsB,EAAE,CAAC;IAC5D,KAAK,EAAE,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE;QACnC,IAAI,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC1D,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,MAAM,gBAAgB,GACpB,MAAA,MAAA,MAAA,MAAA,SAAS,CAAC,iBAAiB,0CAAE,IAAI,0CAAE,UAAU,0CAAE,KAAK,0CAAE,UAAU,CAAC;QACnE,IAAI,gBAAgB,KAAK,SAAS,EAAE;YAClC,MAAM,CAAC,IAAI,CACT,GAAG,IAAI,uDAAuD,CAC/D,CAAC;YACF,SAAS;SACV;QACD,KAAK,MAAM,WAAW,IAAI,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE;YACnD,IAAI,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,EAAE;gBACtC,MAAM,CAAC,IAAI,CACT,GAAG,IAAI,2CAA2C,WAAW,UAAU,CACxE,CAAC;gBACF,SAAS,KAAK,CAAC;aAChB;SACF;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KACnB;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAjCD,sEAiCC;AAED,KAAK,UAAU,QAAQ,CACrB,MAAc,EACd,QAAkB,EAClB,OAAe;IAEf,OAAO,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC;AAC7D,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAc,EACd,QAAkB;IAElB,OAAO,eAAe,aAAa,IAAI,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,QAAQ,GAAG,CAAC;AAClF,CAAC"}
|
||||
192
lib/trap-caching.test.js
generated
Normal file
192
lib/trap-caching.test.js
generated
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const cache = __importStar(require("@actions/cache"));
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const sinon = __importStar(require("sinon"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const codeql_1 = require("./codeql");
|
||||
const languages_1 = require("./languages");
|
||||
const testing_utils_1 = require("./testing-utils");
|
||||
const trap_caching_1 = require("./trap-caching");
|
||||
const util = __importStar(require("./util"));
|
||||
(0, testing_utils_1.setupTests)(ava_1.default);
|
||||
const stubCodeql = (0, codeql_1.setCodeQL)({
|
||||
async getVersion() {
|
||||
return "2.10.3";
|
||||
},
|
||||
async betterResolveLanguages() {
|
||||
return {
|
||||
extractors: {
|
||||
[languages_1.Language.javascript]: [
|
||||
{
|
||||
extractor_root: "some_root",
|
||||
extractor_options: {
|
||||
trap: {
|
||||
properties: {
|
||||
cache: {
|
||||
properties: {
|
||||
dir: {
|
||||
title: "Cache directory",
|
||||
},
|
||||
bound: {
|
||||
title: "Cache bound",
|
||||
},
|
||||
write: {
|
||||
title: "Cache write",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[languages_1.Language.cpp]: [
|
||||
{
|
||||
extractor_root: "other_root",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
const testConfigWithoutTmpDir = {
|
||||
languages: [languages_1.Language.javascript, languages_1.Language.cpp],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: "",
|
||||
codeQLCmd: "",
|
||||
gitHubVersion: {
|
||||
type: util.GitHubVariant.DOTCOM,
|
||||
},
|
||||
dbLocation: "",
|
||||
packs: {},
|
||||
debugMode: false,
|
||||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {
|
||||
javascript: "/some/cache/dir",
|
||||
},
|
||||
};
|
||||
function getTestConfigWithTempDir(tmpDir) {
|
||||
return {
|
||||
languages: [languages_1.Language.javascript, languages_1.Language.ruby],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: tmpDir,
|
||||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM },
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
debugMode: false,
|
||||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {
|
||||
javascript: path.resolve(tmpDir, "jsCache"),
|
||||
ruby: path.resolve(tmpDir, "rubyCache"),
|
||||
},
|
||||
};
|
||||
}
|
||||
(0, ava_1.default)("check flags for JS, analyzing default branch", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfigWithTempDir(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
|
||||
const result = await (0, trap_caching_1.getTrapCachingExtractorConfigArgsForLang)(config, languages_1.Language.javascript);
|
||||
t.deepEqual(result, [
|
||||
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
|
||||
"-O=javascript.trap.cache.bound=1024",
|
||||
"-O=javascript.trap.cache.write=true",
|
||||
]);
|
||||
});
|
||||
});
|
||||
(0, ava_1.default)("check flags for all, not analyzing default branch", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfigWithTempDir(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
|
||||
const result = await (0, trap_caching_1.getTrapCachingExtractorConfigArgs)(config);
|
||||
t.deepEqual(result, [
|
||||
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
|
||||
"-O=javascript.trap.cache.bound=1024",
|
||||
"-O=javascript.trap.cache.write=false",
|
||||
`-O=ruby.trap.cache.dir=${path.resolve(tmpDir, "rubyCache")}`,
|
||||
"-O=ruby.trap.cache.bound=1024",
|
||||
"-O=ruby.trap.cache.write=false",
|
||||
]);
|
||||
});
|
||||
});
|
||||
(0, ava_1.default)("get languages that support TRAP caching", async (t) => {
|
||||
const loggedMessages = [];
|
||||
const logger = (0, testing_utils_1.getRecordingLogger)(loggedMessages);
|
||||
const languagesSupportingCaching = await (0, trap_caching_1.getLanguagesSupportingCaching)(stubCodeql, [languages_1.Language.javascript, languages_1.Language.cpp], logger);
|
||||
t.deepEqual(languagesSupportingCaching, [languages_1.Language.javascript]);
|
||||
});
|
||||
(0, ava_1.default)("upload cache key contains right fields", async (t) => {
|
||||
const loggedMessages = [];
|
||||
const logger = (0, testing_utils_1.getRecordingLogger)(loggedMessages);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
|
||||
const stubSave = sinon.stub(cache, "saveCache");
|
||||
process.env.GITHUB_SHA = "somesha";
|
||||
await (0, trap_caching_1.uploadTrapCaches)(stubCodeql, testConfigWithoutTmpDir, logger);
|
||||
t.assert(stubSave.calledOnceWith(sinon.match.array.contains(["/some/cache/dir"]), sinon
|
||||
.match("somesha")
|
||||
.and(sinon.match("2.10.3"))
|
||||
.and(sinon.match("javascript"))));
|
||||
});
|
||||
(0, ava_1.default)("download cache looks for the right key and creates dir", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages = [];
|
||||
const logger = (0, testing_utils_1.getRecordingLogger)(loggedMessages);
|
||||
sinon.stub(actionsUtil, "getTemporaryDirectory").returns(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
|
||||
const stubRestore = sinon.stub(cache, "restoreCache").resolves("found");
|
||||
const eventFile = path.resolve(tmpDir, "event.json");
|
||||
process.env.GITHUB_EVENT_NAME = "pull_request";
|
||||
process.env.GITHUB_EVENT_PATH = eventFile;
|
||||
fs.writeFileSync(eventFile, JSON.stringify({
|
||||
pull_request: {
|
||||
base: {
|
||||
sha: "somesha",
|
||||
},
|
||||
},
|
||||
}));
|
||||
await (0, trap_caching_1.downloadTrapCaches)(stubCodeql, [languages_1.Language.javascript, languages_1.Language.cpp], logger);
|
||||
t.assert(stubRestore.calledOnceWith(sinon.match.array.contains([
|
||||
path.resolve(tmpDir, "trapCaches", "javascript"),
|
||||
]), sinon
|
||||
.match("somesha")
|
||||
.and(sinon.match("2.10.3"))
|
||||
.and(sinon.match("javascript"))));
|
||||
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=trap-caching.test.js.map
|
||||
1
lib/trap-caching.test.js.map
Normal file
1
lib/trap-caching.test.js.map
Normal file
File diff suppressed because one or more lines are too long
1
lib/util.test.js
generated
1
lib/util.test.js
generated
|
|
@ -258,6 +258,7 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -25,6 +25,7 @@ test("emptyPaths", async (t) => {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
@ -50,6 +51,7 @@ test("nonEmptyPaths", async (t) => {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], "path1\npath2");
|
||||
|
|
@ -78,6 +80,7 @@ test("exclude temp dir", async (t) => {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
analysisPaths.includeAndExcludeAnalysisPaths(config);
|
||||
t.is(process.env["LGTM_INDEX_INCLUDE"], undefined);
|
||||
|
|
|
|||
|
|
@ -5,8 +5,8 @@ import {
|
|||
CodeQLAnalysisError,
|
||||
QueriesStatusReport,
|
||||
runCleanup,
|
||||
runQueries,
|
||||
runFinalize,
|
||||
runQueries,
|
||||
} from "./analyze";
|
||||
import { getGitHubVersionActionsOnly } from "./api-client";
|
||||
import { getCodeQL } from "./codeql";
|
||||
|
|
@ -15,6 +15,7 @@ import { uploadDatabases } from "./database-upload";
|
|||
import { GitHubFeatureFlags } from "./feature-flags";
|
||||
import { getActionsLogger } from "./logging";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import { uploadTrapCaches } from "./trap-caching";
|
||||
import * as upload_lib from "./upload-lib";
|
||||
import { UploadResult } from "./upload-lib";
|
||||
import * as util from "./util";
|
||||
|
|
@ -160,6 +161,9 @@ async function run() {
|
|||
// Possibly upload the database bundles for remote queries
|
||||
await uploadDatabases(repositoryNwo, config, apiDetails, logger);
|
||||
|
||||
// Possibly upload the TRAP caches for later re-use
|
||||
await uploadTrapCaches(codeql, config, logger);
|
||||
|
||||
// We don't upload results in test mode, so don't wait for processing
|
||||
if (util.isInTestMode()) {
|
||||
core.debug("In test mode. Waiting for processing is disabled.");
|
||||
|
|
|
|||
|
|
@ -113,6 +113,7 @@ test("status report fields and search path setting", async (t) => {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
|
||||
recursive: true,
|
||||
|
|
@ -268,6 +269,7 @@ const stubConfig: Config = {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
|
||||
for (const options of [
|
||||
|
|
|
|||
|
|
@ -137,11 +137,7 @@ export async function createdDBForScannedLanguages(
|
|||
await setupPythonExtractor(logger);
|
||||
}
|
||||
|
||||
await codeql.extractScannedLanguage(
|
||||
util.getCodeQLDatabasePath(config, language),
|
||||
language,
|
||||
featureFlags
|
||||
);
|
||||
await codeql.extractScannedLanguage(config, language, featureFlags);
|
||||
logger.endGroup();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -435,6 +435,7 @@ const stubConfig: Config = {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
|
||||
test("databaseInitCluster() Lua feature flag enabled, but old CLI", async (t) => {
|
||||
|
|
|
|||
|
|
@ -9,15 +9,19 @@ import { default as queryString } from "query-string";
|
|||
import * as semver from "semver";
|
||||
import { v4 as uuidV4 } from "uuid";
|
||||
|
||||
import { isRunningLocalAction, getRelativeScriptPath } from "./actions-util";
|
||||
import { getRelativeScriptPath, isRunningLocalAction } from "./actions-util";
|
||||
import * as api from "./api-client";
|
||||
import { Config } from "./config-utils";
|
||||
import * as defaults from "./defaults.json"; // Referenced from codeql-action-sync-tool!
|
||||
import { errorMatchers } from "./error-matcher";
|
||||
import { FeatureFlags, FeatureFlag } from "./feature-flags";
|
||||
import { FeatureFlag, FeatureFlags } from "./feature-flags";
|
||||
import { isTracedLanguage, Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { toolrunnerErrorCatcher } from "./toolrunner-error-catcher";
|
||||
import {
|
||||
getTrapCachingExtractorConfigArgs,
|
||||
getTrapCachingExtractorConfigArgsForLang,
|
||||
} from "./trap-caching";
|
||||
import * as util from "./util";
|
||||
import { isGoodVersion } from "./util";
|
||||
|
||||
|
|
@ -97,7 +101,7 @@ export interface CodeQL {
|
|||
* and running the language extractor.
|
||||
*/
|
||||
extractScannedLanguage(
|
||||
database: string,
|
||||
config: Config,
|
||||
language: Language,
|
||||
featureFlags: FeatureFlags
|
||||
): Promise<void>;
|
||||
|
|
@ -113,6 +117,10 @@ export interface CodeQL {
|
|||
* Run 'codeql resolve languages'.
|
||||
*/
|
||||
resolveLanguages(): Promise<ResolveLanguagesOutput>;
|
||||
/**
|
||||
* Run 'codeql resolve languages' with '--format=betterjson'.
|
||||
*/
|
||||
betterResolveLanguages(): Promise<BetterResolveLanguagesOutput>;
|
||||
/**
|
||||
* Run 'codeql resolve queries'.
|
||||
*/
|
||||
|
|
@ -170,6 +178,17 @@ export interface ResolveLanguagesOutput {
|
|||
[language: string]: [string];
|
||||
}
|
||||
|
||||
export interface BetterResolveLanguagesOutput {
|
||||
extractors: {
|
||||
[language: string]: [
|
||||
{
|
||||
extractor_root: string;
|
||||
extractor_options?: any;
|
||||
}
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
export interface ResolveQueriesOutput {
|
||||
byLanguage: {
|
||||
[language: string]: {
|
||||
|
|
@ -248,6 +267,12 @@ export const CODEQL_VERSION_NEW_TRACING = "2.7.0";
|
|||
*/
|
||||
export const CODEQL_VERSION_ML_POWERED_QUERIES_WINDOWS = "2.9.0";
|
||||
|
||||
/**
|
||||
* Previous versions had the option already, but were missing the
|
||||
* --extractor-options-verbosity that we need.
|
||||
*/
|
||||
export const CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = "2.10.3";
|
||||
|
||||
function getCodeQLBundleName(): string {
|
||||
let platform: string;
|
||||
if (process.platform === "win32") {
|
||||
|
|
@ -585,6 +610,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
|
|||
),
|
||||
finalizeDatabase: resolveFunction(partialCodeql, "finalizeDatabase"),
|
||||
resolveLanguages: resolveFunction(partialCodeql, "resolveLanguages"),
|
||||
betterResolveLanguages: resolveFunction(
|
||||
partialCodeql,
|
||||
"betterResolveLanguages"
|
||||
),
|
||||
resolveQueries: resolveFunction(partialCodeql, "resolveQueries"),
|
||||
packDownload: resolveFunction(partialCodeql, "packDownload"),
|
||||
databaseCleanup: resolveFunction(partialCodeql, "databaseCleanup"),
|
||||
|
|
@ -730,6 +759,7 @@ async function getCodeQLForCmd(
|
|||
);
|
||||
if (config.languages.filter(isTracedLanguage).length > 0) {
|
||||
extraArgs.push("--begin-tracing");
|
||||
extraArgs.push(...(await getTrapCachingExtractorConfigArgs(config)));
|
||||
if (processName !== undefined) {
|
||||
extraArgs.push(`--trace-process-name=${processName}`);
|
||||
} else {
|
||||
|
|
@ -797,10 +827,11 @@ async function getCodeQLForCmd(
|
|||
await runTool(autobuildCmd);
|
||||
},
|
||||
async extractScannedLanguage(
|
||||
databasePath: string,
|
||||
config: Config,
|
||||
language: Language,
|
||||
featureFlags: FeatureFlags
|
||||
) {
|
||||
const databasePath = util.getCodeQLDatabasePath(config, language);
|
||||
// Get extractor location
|
||||
let extractorPath = "";
|
||||
await new toolrunner.ToolRunner(
|
||||
|
|
@ -850,6 +881,7 @@ async function getCodeQLForCmd(
|
|||
"database",
|
||||
"trace-command",
|
||||
...extraArgs,
|
||||
...(await getTrapCachingExtractorConfigArgsForLang(config, language)),
|
||||
...getExtraOptionsFromEnv(["database", "trace-command"]),
|
||||
databasePath,
|
||||
"--",
|
||||
|
|
@ -892,6 +924,24 @@ async function getCodeQLForCmd(
|
|||
);
|
||||
}
|
||||
},
|
||||
async betterResolveLanguages() {
|
||||
const codeqlArgs = [
|
||||
"resolve",
|
||||
"languages",
|
||||
"--format=betterjson",
|
||||
"--extractor-options-verbosity=4",
|
||||
...getExtraOptionsFromEnv(["resolve", "languages"]),
|
||||
];
|
||||
const output = await runTool(cmd, codeqlArgs);
|
||||
|
||||
try {
|
||||
return JSON.parse(output);
|
||||
} catch (e) {
|
||||
throw new Error(
|
||||
`Unexpected output from codeql resolve languages with --format=betterjson: ${e}`
|
||||
);
|
||||
}
|
||||
},
|
||||
async resolveQueries(
|
||||
queries: string[],
|
||||
extraSearchPath: string | undefined
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ test("load empty config", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -106,6 +107,7 @@ test("load empty config", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -151,6 +153,7 @@ test("loading config saves config", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -185,6 +188,7 @@ test("load input outside of workspace", async (t) => {
|
|||
"../input",
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -223,6 +227,7 @@ test("load non-local input with invalid repo syntax", async (t) => {
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -262,6 +267,7 @@ test("load non-existent input", async (t) => {
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -350,6 +356,7 @@ test("load non-empty input", async (t) => {
|
|||
debugArtifactName: "my-artifact",
|
||||
debugDatabaseName: "my-db",
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
|
||||
const languages = "javascript";
|
||||
|
|
@ -362,6 +369,7 @@ test("load non-empty input", async (t) => {
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"my-artifact",
|
||||
"my-db",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -428,6 +436,7 @@ test("Default queries are used", async (t) => {
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -502,6 +511,7 @@ test("Queries can be specified in config file", async (t) => {
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -575,6 +585,7 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -646,6 +657,7 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -708,6 +720,7 @@ test("Multiple queries can be specified in workflow file, no config file require
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -791,6 +804,7 @@ test("Queries in workflow file can be added to the set of queries without overri
|
|||
configFilePath,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -868,6 +882,7 @@ test("Invalid queries in workflow file handled correctly", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -935,6 +950,7 @@ test("API client used when reading remote config", async (t) => {
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -964,6 +980,7 @@ test("Remote config handles the case where a directory is provided", async (t) =
|
|||
repoReference,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1001,6 +1018,7 @@ test("Invalid format of remote config handled correctly", async (t) => {
|
|||
repoReference,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1039,6 +1057,7 @@ test("No detected languages", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1069,6 +1088,7 @@ test("Unknown languages", async (t) => {
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1121,6 +1141,7 @@ test("Config specifies packages", async (t) => {
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1177,6 +1198,7 @@ test("Config specifies packages for multiple languages", async (t) => {
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example" },
|
||||
|
|
@ -1244,6 +1266,7 @@ function doInvalidInputTest(
|
|||
configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
@ -1766,6 +1789,7 @@ const mlPoweredQueriesMacro = test.macro({
|
|||
undefined,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
{ owner: "github", repo: "example " },
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import { FeatureFlag, FeatureFlags } from "./feature-flags";
|
|||
import { Language, parseLanguage } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { RepositoryNwo } from "./repository";
|
||||
import { downloadTrapCaches } from "./trap-caching";
|
||||
import {
|
||||
codeQlVersionAbove,
|
||||
getMlPoweredJsQueriesPack,
|
||||
|
|
@ -148,6 +149,11 @@ export interface Config {
|
|||
* Whether we injected ML queries into this configuration.
|
||||
*/
|
||||
injectedMlQueries: boolean;
|
||||
/**
|
||||
* Partial map from languages to locations of TRAP caches for that language.
|
||||
* If a key is omitted, then TRAP caching should not be used for that language.
|
||||
*/
|
||||
trapCaches: Partial<Record<Language, string>>;
|
||||
}
|
||||
|
||||
export type Packs = Partial<Record<Language, string[]>>;
|
||||
|
|
@ -878,6 +884,7 @@ export async function getDefaultConfig(
|
|||
queriesInput: string | undefined,
|
||||
packsInput: string | undefined,
|
||||
dbLocation: string | undefined,
|
||||
trapCachingEnabled: boolean,
|
||||
debugMode: boolean,
|
||||
debugArtifactName: string,
|
||||
debugDatabaseName: string,
|
||||
|
|
@ -937,6 +944,9 @@ export async function getDefaultConfig(
|
|||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
injectedMlQueries,
|
||||
trapCaches: trapCachingEnabled
|
||||
? await downloadTrapCaches(codeQL, languages, logger)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -949,6 +959,7 @@ async function loadConfig(
|
|||
packsInput: string | undefined,
|
||||
configFile: string,
|
||||
dbLocation: string | undefined,
|
||||
trapCachingEnabled: boolean,
|
||||
debugMode: boolean,
|
||||
debugArtifactName: string,
|
||||
debugDatabaseName: string,
|
||||
|
|
@ -1117,6 +1128,9 @@ async function loadConfig(
|
|||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
injectedMlQueries,
|
||||
trapCaches: trapCachingEnabled
|
||||
? await downloadTrapCaches(codeQL, languages, logger)
|
||||
: {},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -1358,6 +1372,7 @@ export async function initConfig(
|
|||
packsInput: string | undefined,
|
||||
configFile: string | undefined,
|
||||
dbLocation: string | undefined,
|
||||
trapCachingEnabled: boolean,
|
||||
debugMode: boolean,
|
||||
debugArtifactName: string,
|
||||
debugDatabaseName: string,
|
||||
|
|
@ -1380,6 +1395,7 @@ export async function initConfig(
|
|||
queriesInput,
|
||||
packsInput,
|
||||
dbLocation,
|
||||
trapCachingEnabled,
|
||||
debugMode,
|
||||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
|
|
@ -1399,6 +1415,7 @@ export async function initConfig(
|
|||
packsInput,
|
||||
configFile,
|
||||
dbLocation,
|
||||
trapCachingEnabled,
|
||||
debugMode,
|
||||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ function getTestConfig(tmpDir: string): Config {
|
|||
debugArtifactName: DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ export interface FeatureFlags {
|
|||
export enum FeatureFlag {
|
||||
MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
|
||||
LuaTracerConfigEnabled = "lua_tracer_config_enabled",
|
||||
TrapCachingEnabled = "trap_caching_enabled",
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import {
|
|||
import { getGitHubVersionActionsOnly } from "./api-client";
|
||||
import { CodeQL, CODEQL_VERSION_NEW_TRACING } from "./codeql";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { GitHubFeatureFlags } from "./feature-flags";
|
||||
import { FeatureFlag, FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
|
||||
import {
|
||||
initCodeQL,
|
||||
initConfig,
|
||||
|
|
@ -27,18 +27,18 @@ import { Language } from "./languages";
|
|||
import { getActionsLogger } from "./logging";
|
||||
import { parseRepositoryNwo } from "./repository";
|
||||
import {
|
||||
getRequiredEnvParam,
|
||||
initializeEnvironment,
|
||||
Mode,
|
||||
checkActionVersion,
|
||||
checkGitHubVersionInRange,
|
||||
codeQlVersionAbove,
|
||||
enrichEnvironment,
|
||||
getMemoryFlagValue,
|
||||
getThreadsFlagValue,
|
||||
DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
DEFAULT_DEBUG_DATABASE_NAME,
|
||||
enrichEnvironment,
|
||||
getMemoryFlagValue,
|
||||
getMlPoweredJsQueriesStatus,
|
||||
checkActionVersion,
|
||||
getRequiredEnvParam,
|
||||
getThreadsFlagValue,
|
||||
initializeEnvironment,
|
||||
Mode,
|
||||
} from "./util";
|
||||
|
||||
// eslint-disable-next-line import/no-commonjs
|
||||
|
|
@ -183,6 +183,7 @@ async function run() {
|
|||
getOptionalInput("packs"),
|
||||
getOptionalInput("config-file"),
|
||||
getOptionalInput("db-location"),
|
||||
await getTrapCachingEnabled(featureFlags),
|
||||
// Debug mode is enabled if:
|
||||
// - The `init` Action is passed `debug: true`.
|
||||
// - Actions step debugging is enabled (e.g. by [enabling debug logging for a rerun](https://docs.github.com/en/actions/managing-workflow-runs/re-running-workflows-and-jobs#re-running-all-the-jobs-in-a-workflow),
|
||||
|
|
@ -299,6 +300,14 @@ async function run() {
|
|||
await sendSuccessStatusReport(startedAt, config, toolsVersion);
|
||||
}
|
||||
|
||||
async function getTrapCachingEnabled(
|
||||
featureFlags: FeatureFlags
|
||||
): Promise<boolean> {
|
||||
const trapCaching = getOptionalInput("trap-caching");
|
||||
if (trapCaching !== undefined) return trapCaching === "true";
|
||||
return await featureFlags.getValue(FeatureFlag.TrapCachingEnabled);
|
||||
}
|
||||
|
||||
async function runWrapper() {
|
||||
try {
|
||||
await run();
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ export async function initConfig(
|
|||
packsInput: string | undefined,
|
||||
configFile: string | undefined,
|
||||
dbLocation: string | undefined,
|
||||
trapCachingEnabled: boolean,
|
||||
debugMode: boolean,
|
||||
debugArtifactName: string,
|
||||
debugDatabaseName: string,
|
||||
|
|
@ -61,6 +62,7 @@ export async function initConfig(
|
|||
packsInput,
|
||||
configFile,
|
||||
dbLocation,
|
||||
trapCachingEnabled,
|
||||
debugMode,
|
||||
debugArtifactName,
|
||||
debugDatabaseName,
|
||||
|
|
|
|||
|
|
@ -241,6 +241,7 @@ program
|
|||
cmd.configFile,
|
||||
undefined,
|
||||
false,
|
||||
false,
|
||||
"",
|
||||
"",
|
||||
parseRepositoryNwo(cmd.repository),
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ function getTestConfig(tmpDir: string): configUtils.Config {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
210
src/trap-caching.test.ts
Normal file
210
src/trap-caching.test.ts
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as cache from "@actions/cache";
|
||||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { setCodeQL } from "./codeql";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { Config } from "./config-utils";
|
||||
import { Language } from "./languages";
|
||||
import { getRecordingLogger, setupTests } from "./testing-utils";
|
||||
import {
|
||||
downloadTrapCaches,
|
||||
getLanguagesSupportingCaching,
|
||||
getTrapCachingExtractorConfigArgs,
|
||||
getTrapCachingExtractorConfigArgsForLang,
|
||||
uploadTrapCaches,
|
||||
} from "./trap-caching";
|
||||
import * as util from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
const stubCodeql = setCodeQL({
|
||||
async getVersion() {
|
||||
return "2.10.3";
|
||||
},
|
||||
async betterResolveLanguages() {
|
||||
return {
|
||||
extractors: {
|
||||
[Language.javascript]: [
|
||||
{
|
||||
extractor_root: "some_root",
|
||||
extractor_options: {
|
||||
trap: {
|
||||
properties: {
|
||||
cache: {
|
||||
properties: {
|
||||
dir: {
|
||||
title: "Cache directory",
|
||||
},
|
||||
bound: {
|
||||
title: "Cache bound",
|
||||
},
|
||||
write: {
|
||||
title: "Cache write",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
[Language.cpp]: [
|
||||
{
|
||||
extractor_root: "other_root",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const testConfigWithoutTmpDir: Config = {
|
||||
languages: [Language.javascript, Language.cpp],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: "",
|
||||
codeQLCmd: "",
|
||||
gitHubVersion: {
|
||||
type: util.GitHubVariant.DOTCOM,
|
||||
} as util.GitHubVersion,
|
||||
dbLocation: "",
|
||||
packs: {},
|
||||
debugMode: false,
|
||||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {
|
||||
javascript: "/some/cache/dir",
|
||||
},
|
||||
};
|
||||
|
||||
function getTestConfigWithTempDir(tmpDir: string): configUtils.Config {
|
||||
return {
|
||||
languages: [Language.javascript, Language.ruby],
|
||||
queries: {},
|
||||
pathsIgnore: [],
|
||||
paths: [],
|
||||
originalUserInput: {},
|
||||
tempDir: tmpDir,
|
||||
codeQLCmd: "",
|
||||
gitHubVersion: { type: util.GitHubVariant.DOTCOM } as util.GitHubVersion,
|
||||
dbLocation: path.resolve(tmpDir, "codeql_databases"),
|
||||
packs: {},
|
||||
debugMode: false,
|
||||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {
|
||||
javascript: path.resolve(tmpDir, "jsCache"),
|
||||
ruby: path.resolve(tmpDir, "rubyCache"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("check flags for JS, analyzing default branch", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfigWithTempDir(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
|
||||
const result = await getTrapCachingExtractorConfigArgsForLang(
|
||||
config,
|
||||
Language.javascript
|
||||
);
|
||||
t.deepEqual(result, [
|
||||
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
|
||||
"-O=javascript.trap.cache.bound=1024",
|
||||
"-O=javascript.trap.cache.write=true",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("check flags for all, not analyzing default branch", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const config = getTestConfigWithTempDir(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
|
||||
const result = await getTrapCachingExtractorConfigArgs(config);
|
||||
t.deepEqual(result, [
|
||||
`-O=javascript.trap.cache.dir=${path.resolve(tmpDir, "jsCache")}`,
|
||||
"-O=javascript.trap.cache.bound=1024",
|
||||
"-O=javascript.trap.cache.write=false",
|
||||
`-O=ruby.trap.cache.dir=${path.resolve(tmpDir, "rubyCache")}`,
|
||||
"-O=ruby.trap.cache.bound=1024",
|
||||
"-O=ruby.trap.cache.write=false",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test("get languages that support TRAP caching", async (t) => {
|
||||
const loggedMessages = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
const languagesSupportingCaching = await getLanguagesSupportingCaching(
|
||||
stubCodeql,
|
||||
[Language.javascript, Language.cpp],
|
||||
logger
|
||||
);
|
||||
t.deepEqual(languagesSupportingCaching, [Language.javascript]);
|
||||
});
|
||||
|
||||
test("upload cache key contains right fields", async (t) => {
|
||||
const loggedMessages = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
|
||||
const stubSave = sinon.stub(cache, "saveCache");
|
||||
process.env.GITHUB_SHA = "somesha";
|
||||
await uploadTrapCaches(stubCodeql, testConfigWithoutTmpDir, logger);
|
||||
t.assert(
|
||||
stubSave.calledOnceWith(
|
||||
sinon.match.array.contains(["/some/cache/dir"]),
|
||||
sinon
|
||||
.match("somesha")
|
||||
.and(sinon.match("2.10.3"))
|
||||
.and(sinon.match("javascript"))
|
||||
)
|
||||
);
|
||||
});
|
||||
|
||||
test("download cache looks for the right key and creates dir", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
const loggedMessages = [];
|
||||
const logger = getRecordingLogger(loggedMessages);
|
||||
sinon.stub(actionsUtil, "getTemporaryDirectory").returns(tmpDir);
|
||||
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
|
||||
const stubRestore = sinon.stub(cache, "restoreCache").resolves("found");
|
||||
const eventFile = path.resolve(tmpDir, "event.json");
|
||||
process.env.GITHUB_EVENT_NAME = "pull_request";
|
||||
process.env.GITHUB_EVENT_PATH = eventFile;
|
||||
fs.writeFileSync(
|
||||
eventFile,
|
||||
JSON.stringify({
|
||||
pull_request: {
|
||||
base: {
|
||||
sha: "somesha",
|
||||
},
|
||||
},
|
||||
})
|
||||
);
|
||||
await downloadTrapCaches(
|
||||
stubCodeql,
|
||||
[Language.javascript, Language.cpp],
|
||||
logger
|
||||
);
|
||||
t.assert(
|
||||
stubRestore.calledOnceWith(
|
||||
sinon.match.array.contains([
|
||||
path.resolve(tmpDir, "trapCaches", "javascript"),
|
||||
]),
|
||||
sinon
|
||||
.match("somesha")
|
||||
.and(sinon.match("2.10.3"))
|
||||
.and(sinon.match("javascript"))
|
||||
)
|
||||
);
|
||||
t.assert(fs.existsSync(path.resolve(tmpDir, "trapCaches", "javascript")));
|
||||
});
|
||||
});
|
||||
191
src/trap-caching.ts
Normal file
191
src/trap-caching.ts
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
import * as cache from "@actions/cache";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import { CodeQL, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES } from "./codeql";
|
||||
import { Config } from "./config-utils";
|
||||
import { Language } from "./languages";
|
||||
import { Logger } from "./logging";
|
||||
import { codeQlVersionAbove } from "./util";
|
||||
|
||||
// This constant should be bumped if we make a breaking change
|
||||
// to how the CodeQL Action stores or retrieves the TRAP cache,
|
||||
// and will invalidate previous caches. We don't need to bump
|
||||
// this for CLI/extractor changes, since the CLI version also
|
||||
// goes into the cache key.
|
||||
const CACHE_VERSION = 1;
|
||||
|
||||
// This constant sets the size of each TRAP cache in megabytes.
|
||||
const CACHE_SIZE_MB = 1024;
|
||||
|
||||
export async function getTrapCachingExtractorConfigArgs(
|
||||
config: Config
|
||||
): Promise<string[]> {
|
||||
const result: string[][] = [];
|
||||
for (const language of config.languages)
|
||||
result.push(
|
||||
await getTrapCachingExtractorConfigArgsForLang(config, language)
|
||||
);
|
||||
return result.flat();
|
||||
}
|
||||
|
||||
export async function getTrapCachingExtractorConfigArgsForLang(
|
||||
config: Config,
|
||||
language: Language
|
||||
): Promise<string[]> {
|
||||
const cacheDir = config.trapCaches[language];
|
||||
if (cacheDir === undefined) return [];
|
||||
const write = await actionsUtil.isAnalyzingDefaultBranch();
|
||||
return [
|
||||
`-O=${language}.trap.cache.dir=${cacheDir}`,
|
||||
`-O=${language}.trap.cache.bound=${CACHE_SIZE_MB}`,
|
||||
`-O=${language}.trap.cache.write=${write}`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Download TRAP caches from the Actions cache.
|
||||
* @param codeql The CodeQL instance to use.
|
||||
* @param languages The languages being analyzed.
|
||||
* @param logger A logger to record some informational messages to.
|
||||
* @returns A partial map from languages to TRAP cache paths on disk, with
|
||||
* languages for which we shouldn't use TRAP caching omitted.
|
||||
*/
|
||||
export async function downloadTrapCaches(
|
||||
codeql: CodeQL,
|
||||
languages: Language[],
|
||||
logger: Logger
|
||||
): Promise<Partial<Record<Language, string>>> {
|
||||
const result = {};
|
||||
const languagesSupportingCaching = await getLanguagesSupportingCaching(
|
||||
codeql,
|
||||
languages,
|
||||
logger
|
||||
);
|
||||
logger.info(
|
||||
`Found ${languagesSupportingCaching.length} languages that support TRAP caching`
|
||||
);
|
||||
if (languagesSupportingCaching.length === 0) return result;
|
||||
|
||||
const cachesDir = path.join(
|
||||
actionsUtil.getTemporaryDirectory(),
|
||||
"trapCaches"
|
||||
);
|
||||
for (const language of languagesSupportingCaching) {
|
||||
const cacheDir = path.join(cachesDir, language);
|
||||
fs.mkdirSync(cacheDir, { recursive: true });
|
||||
result[language] = cacheDir;
|
||||
}
|
||||
|
||||
if (await actionsUtil.isAnalyzingDefaultBranch()) {
|
||||
logger.info(
|
||||
"Analyzing default branch. Skipping downloading of TRAP caches."
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
let baseSha = "unknown";
|
||||
const eventPath = process.env.GITHUB_EVENT_PATH;
|
||||
if (
|
||||
process.env.GITHUB_EVENT_NAME === "pull_request" &&
|
||||
eventPath !== undefined
|
||||
) {
|
||||
const event = JSON.parse(fs.readFileSync(path.resolve(eventPath), "utf-8"));
|
||||
baseSha = event.pull_request?.base?.sha || baseSha;
|
||||
}
|
||||
for (const language of languages) {
|
||||
const cacheDir = result[language];
|
||||
if (cacheDir === undefined) continue;
|
||||
// The SHA from the base of the PR is the most similar commit we might have a cache for
|
||||
const preferredKey = await cacheKey(codeql, language, baseSha);
|
||||
logger.info(
|
||||
`Looking in Actions cache for TRAP cache with key ${preferredKey}`
|
||||
);
|
||||
const found = await cache.restoreCache([cacheDir], preferredKey, [
|
||||
await cachePrefix(codeql, language), // Fall back to any cache with the right key prefix
|
||||
]);
|
||||
if (found === undefined) {
|
||||
// We didn't find a TRAP cache in the Actions cache, so the directory on disk is
|
||||
// still just an empty directory. There's no reason to tell the extractor to use it,
|
||||
// so let's unset the entry in the map so we don't set any extractor options.
|
||||
logger.info(`No TRAP cache found in Actions cache for ${language}`);
|
||||
result[language] = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function uploadTrapCaches(
|
||||
codeql: CodeQL,
|
||||
config: Config,
|
||||
logger: Logger
|
||||
): Promise<void> {
|
||||
if (!(await actionsUtil.isAnalyzingDefaultBranch())) return; // Only upload caches from the default branch
|
||||
|
||||
const toAwait: Array<Promise<number>> = [];
|
||||
for (const language of config.languages) {
|
||||
const cacheDir = config.trapCaches[language];
|
||||
if (cacheDir === undefined) continue;
|
||||
const key = await cacheKey(
|
||||
codeql,
|
||||
language,
|
||||
process.env.GITHUB_SHA || "unknown"
|
||||
);
|
||||
logger.info(`Uploading TRAP cache to Actions cache with key ${key}`);
|
||||
toAwait.push(cache.saveCache([cacheDir], key));
|
||||
}
|
||||
await Promise.all(toAwait);
|
||||
}
|
||||
|
||||
export async function getLanguagesSupportingCaching(
|
||||
codeql: CodeQL,
|
||||
languages: Language[],
|
||||
logger: Logger
|
||||
): Promise<Language[]> {
|
||||
const result: Language[] = [];
|
||||
if (
|
||||
!(await codeQlVersionAbove(codeql, CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES))
|
||||
)
|
||||
return result;
|
||||
const resolveResult = await codeql.betterResolveLanguages();
|
||||
outer: for (const lang of languages) {
|
||||
if (resolveResult.extractors[lang].length !== 1) continue;
|
||||
const extractor = resolveResult.extractors[lang][0];
|
||||
const trapCacheOptions =
|
||||
extractor.extractor_options?.trap?.properties?.cache?.properties;
|
||||
if (trapCacheOptions === undefined) {
|
||||
logger.info(
|
||||
`${lang} does not support TRAP caching (missing option group)`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const requiredOpt of ["dir", "bound", "write"]) {
|
||||
if (!(requiredOpt in trapCacheOptions)) {
|
||||
logger.info(
|
||||
`${lang} does not support TRAP caching (missing ${requiredOpt} option)`
|
||||
);
|
||||
continue outer;
|
||||
}
|
||||
}
|
||||
result.push(lang);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function cacheKey(
|
||||
codeql: CodeQL,
|
||||
language: Language,
|
||||
baseSha: string
|
||||
): Promise<string> {
|
||||
return `${await cachePrefix(codeql, language)}-${baseSha}`;
|
||||
}
|
||||
|
||||
async function cachePrefix(
|
||||
codeql: CodeQL,
|
||||
language: Language
|
||||
): Promise<string> {
|
||||
return `codeql-trap-${CACHE_VERSION}-${await codeql.getVersion()}-${language}-`;
|
||||
}
|
||||
|
|
@ -348,6 +348,7 @@ for (const [packs, expectedStatus] of ML_POWERED_JS_STATUS_TESTS) {
|
|||
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
|
||||
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
|
||||
injectedMlQueries: false,
|
||||
trapCaches: {},
|
||||
};
|
||||
|
||||
t.is(util.getMlPoweredJsQueriesStatus(config), expectedStatus);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue