robojerk/codeql-action

Use new feature flag architecture when uploading databases

Browse source
This commit is contained in:
Henry Mercer 2021-12-14 20:48:32 +00:00
parent 04671efa1d
commit d6499fad61
18 changed files with 158 additions and 177 deletions
Download patch file Download diff file Expand all files Collapse all files

2
lib/analyze-action-env.test.js generated
View file

@ -45,6 +45,8 @@ const util = __importStar(require("./util"));
.resolves({});
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
sinon.stub(configUtils, "getConfig").resolves({
// Use GHES so we don't try to call the feature flags API endpoint
gitHubVersion: { type: util.GitHubVariant.GHES, version: "3.0.0" },
languages: [],
});
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");

2
lib/analyze-action-env.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,kEAAkE;YAClE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE;YAClE,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

2
lib/analyze-action-input.test.js generated
View file

@ -45,6 +45,8 @@ const util = __importStar(require("./util"));
.resolves({});
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
sinon.stub(configUtils, "getConfig").resolves({
// Use GHES so we don't try to call the feature flags API endpoint
gitHubVersion: { type: util.GitHubVariant.GHES, version: "3.0.0" },
languages: [],
});
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");

2
lib/analyze-action-input.test.js.map
View file

@ -1 +1 @@
{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,kEAAkE;YAClE,aAAa,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE;YAClE,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

6
lib/analyze-action.js generated
View file

@ -29,6 +29,7 @@ const analyze_1 = require("./analyze");
const codeql_1 = require("./codeql");
const config_utils_1 = require("./config-utils");
const database_upload_1 = require("./database-upload");
const feature_flags_1 = require("./feature-flags");
const logging_1 = require("./logging");
const repository_1 = require("./repository");
const upload_lib = __importStar(require("./upload-lib"));
@ -71,6 +72,8 @@ async function run() {
const outputDir = actionsUtil.getRequiredInput("output");
const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
const featureFlags = new feature_flags_1.GitHubFeatureFlags(config.gitHubVersion, apiDetails, logger);
void featureFlags.preloadFeatureFlags();
await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
@ -111,7 +114,8 @@ async function run() {
logger.info("Not uploading results");
}
const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, apiDetails, logger); // Possibly upload the database bundles for remote queries
// Possibly upload the database bundles for remote queries
await (0, database_upload_1.uploadDatabases)(repositoryNwo, config, featureFlags, apiDetails, logger);
if (uploadResult !== undefined &&
actionsUtil.getRequiredInput("wait-for-processing") === "true") {
await upload_lib.waitForProcessing((0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY")), uploadResult.sarifID, apiDetails, (0, logging_1.getActionsLogger)());

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

23
lib/database-upload.js generated
View file

@ -26,7 +26,7 @@ const api_client_1 = require("./api-client");
const codeql_1 = require("./codeql");
const util = __importStar(require("./util"));
const util_1 = require("./util");
async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
async function uploadDatabases(repositoryNwo, config, featureFlags, apiDetails, logger) {
if (actionsUtil.getRequiredInput("upload-database") !== "true") {
logger.debug("Database upload disabled in workflow. Skipping upload.");
return;
@ -41,26 +41,13 @@ async function uploadDatabases(repositoryNwo, config, apiDetails, logger) {
logger.debug("Not analyzing default branch. Skipping upload.");
return;
}
const client = (0, api_client_1.getApiClient)(apiDetails);
let useUploadDomain;
try {
const response = await client.request("GET /repos/:owner/:repo/code-scanning/codeql/databases", {
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
});
useUploadDomain = response.data["uploads_domain_enabled"];
}
catch (e) {
if (util.isHTTPError(e) && e.status === 404) {
logger.debug("Repository is not opted in to database uploads. Skipping upload.");
}
else {
console.log(e);
logger.info(`Skipping database upload due to unknown error: ${e}`);
}
if (!(await featureFlags.getDatabaseUploadsEnabled())) {
logger.debug("Repository is not opted in to database uploads. Skipping upload.");
return;
}
const client = (0, api_client_1.getApiClient)(apiDetails);
const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
const useUploadDomain = await featureFlags.getUploadsDomainEnabled();
for (const language of config.languages) {
// Upload the database bundle.
// Although we are uploading arbitrary file contents to the API, it's worth

2
lib/database-upload.js.map
View file

@ -1 +1 @@
{"version":3,"file":"database-upload.js","sourceRoot":"","sources":["../src/database-upload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAEzB,4DAA8C;AAC9C,6CAA8D;AAC9D,qCAAqC;AAIrC,6CAA+B;AAC/B,iCAAkC;AAE3B,KAAK,UAAU,eAAe,CACnC,aAA4B,EAC5B,MAAc,EACd,UAA4B,EAC5B,MAAc;IAEd,IAAI,WAAW,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,MAAM,EAAE;QAC9D,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACvE,OAAO;KACR;IAED,iDAAiD;IACjD,IAAI,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,OAAO;KACR;IAED,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,wBAAwB,EAAE,CAAC,EAAE;QACnD,4EAA4E;QAC5E,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC/D,OAAO;KACR;IAED,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,UAAU,CAAC,CAAC;IACxC,IAAI,eAAwB,CAAC;IAC7B,IAAI;QACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,wDAAwD,EACxD;YACE,KAAK,EAAE,aAAa,CAAC,KAAK;YAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;SACzB,CACF,CAAC;QACF,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;KAC3D;IAAC,OAAO,CAAC,EAAE;QACV,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;YAC3C,MAAM,CAAC,KAAK,CACV,kEAAkE,CACnE,CAAC;SACH;aAAM;YACL,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,EAAE,CAAC,CAAC;SACpE;QACD,OAAO;KACR;IAED,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,8BAA8B;QAC9B,2EAA2E;QAC3E,8EAA8E;QAC9E,wEAAwE;QACxE,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,IAAA,eAAQ,EAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IAAI;YACF,IAAI,eAAe,EAAE;gBACnB,MAAM,MAAM,CAAC,OAAO,CAClB,wGAAwG,EACxG;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,GAAG,QAAQ,WAAW;oBAC5B,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE;wBACP,aAAa,EAAE,SAAS,UAAU,CAAC,IAAI,EAAE;qBAC1C;iBACF,CACF,CAAC;aACH;iBAAM;gBACL,MAAM,MAAM,CAAC,OAAO,CAClB,kEAAkE,EAClE;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,OAAO;iBACd,CACF,CAAC;aACH;YACD,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;SAChE;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,4CAA4C;YAC5C,MAAM,CAAC,OAAO,CAAC,iCAAiC,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;SACnE;KACF;AACH,CAAC;AAtFD,0CAsFC"}
{"version":3,"file":"database-upload.js","sourceRoot":"","sources":["../src/database-upload.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAEzB,4DAA8C;AAC9C,6CAA8D;AAC9D,qCAAqC;AAKrC,6CAA+B;AAC/B,iCAAkC;AAE3B,KAAK,UAAU,eAAe,CACnC,aAA4B,EAC5B,MAAc,EACd,YAA0B,EAC1B,UAA4B,EAC5B,MAAc;IAEd,IAAI,WAAW,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,KAAK,MAAM,EAAE;QAC9D,MAAM,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACvE,OAAO;KACR;IAED,iDAAiD;IACjD,IAAI,MAAM,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;QAC3D,MAAM,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACjE,OAAO;KACR;IAED,IAAI,CAAC,CAAC,MAAM,WAAW,CAAC,wBAAwB,EAAE,CAAC,EAAE;QACnD,4EAA4E;QAC5E,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;QAC/D,OAAO;KACR;IAED,IAAI,CAAC,CAAC,MAAM,YAAY,CAAC,yBAAyB,EAAE,CAAC,EAAE;QACrD,MAAM,CAAC,KAAK,CACV,kEAAkE,CACnE,CAAC;QACF,OAAO;KACR;IAED,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,UAAU,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,MAAM,IAAA,kBAAS,EAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,eAAe,GAAG,MAAM,YAAY,CAAC,uBAAuB,EAAE,CAAC;IAErE,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,8BAA8B;QAC9B,2EAA2E;QAC3E,8EAA8E;QAC9E,wEAAwE;QACxE,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,IAAA,eAAQ,EAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC1E,IAAI;YACF,IAAI,eAAe,EAAE;gBACnB,MAAM,MAAM,CAAC,OAAO,CAClB,wGAAwG,EACxG;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,GAAG,QAAQ,WAAW;oBAC5B,IAAI,EAAE,OAAO;oBACb,OAAO,EAAE;wBACP,aAAa,EAAE,SAAS,UAAU,CAAC,IAAI,EAAE;qBAC1C;iBACF,CACF,CAAC;aACH;iBAAM;gBACL,MAAM,MAAM,CAAC,OAAO,CAClB,kEAAkE,EAClE;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;oBACxB,QAAQ;oBACR,IAAI,EAAE,OAAO;iBACd,CACF,CAAC;aACH;YACD,MAAM,CAAC,KAAK,CAAC,sCAAsC,QAAQ,EAAE,CAAC,CAAC;SAChE;QAAC,OAAO,CAAC,EAAE;YACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACf,4CAA4C;YAC5C,MAAM,CAAC,OAAO,CAAC,iCAAiC,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;SACnE;KACF;AACH,CAAC;AA1ED,0CA0EC"}

85
lib/database-upload.test.js generated
View file

@ -30,6 +30,7 @@ const actionsUtil = __importStar(require("./actions-util"));
const apiClient = __importStar(require("./api-client"));
const codeql_1 = require("./codeql");
const database_upload_1 = require("./database-upload");
const feature_flags_1 = require("./feature-flags");
const languages_1 = require("./languages");
const testing_utils_1 = require("./testing-utils");
const util_1 = require("./util");
@ -37,6 +38,10 @@ const util_1 = require("./util");
ava_1.default.beforeEach(() => {
(0, util_1.initializeEnvironment)(util_1.Mode.actions, "1.2.3");
});
const uploadToUploadsDomainFlags = (0, feature_flags_1.createFeatureFlags)([
"database_uploads_enabled",
"uploads_domain_enabled",
]);
const testRepoName = { owner: "github", repo: "example" };
const testApiDetails = {
auth: "1234",
@ -58,35 +63,19 @@ function getTestConfig(tmpDir) {
debugMode: false,
};
}
function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatusCode) {
async function mockHttpRequests(featureFlags, databaseUploadStatusCode) {
// Passing an auth token is required, so we just use a dummy value
const client = github.getOctokit("123");
const requestSpy = sinon.stub(client, "request");
const optInSpy = requestSpy.withArgs("GET /repos/:owner/:repo/code-scanning/codeql/databases");
if (optInStatusCode < 300) {
optInSpy.resolves({
status: optInStatusCode,
data: {
useUploadDomain,
},
headers: {},
url: "GET /repos/:owner/:repo/code-scanning/codeql/databases",
});
const url = (await featureFlags.getUploadsDomainEnabled())
? "POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name"
: "PUT /repos/:owner/:repo/code-scanning/codeql/databases/:language";
const databaseUploadSpy = requestSpy.withArgs(url);
if (databaseUploadStatusCode < 300) {
databaseUploadSpy.resolves(undefined);
}
else {
optInSpy.throws(new util_1.HTTPError("some error message", optInStatusCode));
}
if (databaseUploadStatusCode !== undefined) {
const url = useUploadDomain
? "POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name"
: "PUT /repos/:owner/:repo/code-scanning/codeql/databases/:language";
const databaseUploadSpy = requestSpy.withArgs(url);
if (databaseUploadStatusCode < 300) {
databaseUploadSpy.resolves(undefined);
}
else {
databaseUploadSpy.throws(new util_1.HTTPError("some error message", databaseUploadStatusCode));
}
databaseUploadSpy.throws(new util_1.HTTPError("some error message", databaseUploadStatusCode));
}
sinon.stub(apiClient, "getApiClient").value(() => client);
}
@ -99,7 +88,7 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.returns("false");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), uploadToUploadsDomainFlags, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Database upload disabled in workflow. Skipping upload.") !== undefined);
});
@ -115,7 +104,7 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
const config = getTestConfig(tmpDir);
config.gitHubVersion = { type: util_1.GitHubVariant.GHES, version: "3.0" };
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, config, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, config, (0, feature_flags_1.createFeatureFlags)([]), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Not running against github.com. Skipping upload.") !== undefined);
});
@ -131,7 +120,7 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
const config = getTestConfig(tmpDir);
config.gitHubVersion = { type: util_1.GitHubVariant.GHAE };
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, config, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, config, (0, feature_flags_1.createFeatureFlags)([]), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Not running against github.com. Skipping upload.") !== undefined);
});
@ -145,12 +134,12 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(false);
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), uploadToUploadsDomainFlags, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Not analyzing default branch. Skipping upload.") !== undefined);
});
});
(0, ava_1.default)("Abort database upload if opt-in request returns 404", async (t) => {
(0, ava_1.default)("Abort database upload if feature flag is disabled", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
sinon
@ -158,40 +147,18 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(404);
(0, codeql_1.setCodeQL)({
async databaseBundle() {
return;
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), (0, feature_flags_1.createFeatureFlags)(["uploads_domain_enabled"]), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message ===
"Repository is not opted in to database uploads. Skipping upload.") !== undefined);
});
});
(0, ava_1.default)("Abort database upload if opt-in request fails with something other than 404", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(500);
(0, codeql_1.setCodeQL)({
async databaseBundle() {
return;
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "info" &&
v.message ===
"Skipping database upload due to unknown error: Error: some error message") !== undefined);
});
});
(0, ava_1.default)("Don't crash if uploading a database fails", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
@ -200,14 +167,16 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, false, 500);
const featureFlags = (0, feature_flags_1.createFeatureFlags)(["database_uploads_enabled"]);
await mockHttpRequests(featureFlags, 500);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), featureFlags, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
console.log(loggedMessages);
t.assert(loggedMessages.find((v) => v.type === "warning" &&
v.message ===
"Failed to upload database for javascript: Error: some error message") !== undefined);
@ -221,14 +190,14 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, false, 201);
await mockHttpRequests(uploadToUploadsDomainFlags, 201);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), uploadToUploadsDomainFlags, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Successfully uploaded database for javascript") !== undefined);
});
@ -241,14 +210,14 @@ function mockHttpRequests(optInStatusCode, useUploadDomain, databaseUploadStatus
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, true, 201);
await mockHttpRequests(uploadToUploadsDomainFlags, 201);
(0, codeql_1.setCodeQL)({
async databaseBundle(_, outputFilePath) {
fs.writeFileSync(outputFilePath, "");
},
});
const loggedMessages = [];
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
await (0, database_upload_1.uploadDatabases)(testRepoName, getTestConfig(tmpDir), uploadToUploadsDomainFlags, testApiDetails, (0, testing_utils_1.getRecordingLogger)(loggedMessages));
t.assert(loggedMessages.find((v) => v.type === "debug" &&
v.message === "Successfully uploaded database for javascript") !== undefined);
});

2
lib/database-upload.test.js.map
View file

File diff suppressed because one or more lines are too long

21
lib/feature-flags.js generated
View file

@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.GitHubFeatureFlags = void 0;
exports.createFeatureFlags = exports.GitHubFeatureFlags = void 0;
const api_client_1 = require("./api-client");
const repository_1 = require("./repository");
const util = __importStar(require("./util"));
@ -76,4 +76,23 @@ class GitHubFeatureFlags {
}
}
exports.GitHubFeatureFlags = GitHubFeatureFlags;
/**
* Create a feature flags instance with the specified set of enabled flags.
*
* This should be only used within tests.
*/
function createFeatureFlags(enabledFlags) {
return {
getDatabaseUploadsEnabled: async () => {
return enabledFlags.includes("database_uploads_enabled");
},
getMlPoweredQueriesEnabled: async () => {
return enabledFlags.includes("ml_powered_queries_enabled");
},
getUploadsDomainEnabled: async () => {
return enabledFlags.includes("uploads_domain_enabled");
},
};
}
exports.createFeatureFlags = createFeatureFlags;
//# sourceMappingURL=feature-flags.js.map

2
lib/feature-flags.js.map
View file

@ -1 +1 @@
{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAE9D,6CAAkD;AAClD,6CAA+B;AAgB/B,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,MAAc;QAFd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,yBAAyB;QACvB,OAAO,IAAI,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAC;IACzD,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,cAAc,CAAC,4BAA4B,CAAC,CAAC;IAC3D,CAAC;IAED,uBAAuB;QACrB,OAAO,IAAI,CAAC,cAAc,CAAC,wBAAwB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAY;QACvC,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;SACH;QACD,OAAO,QAAQ,IAAI,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;YACF,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;iBACzB,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qDAAqD,CAAC,EAAE,CACzD,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAtED,gDAsEC"}
{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAE9D,6CAAkD;AAClD,6CAA+B;AAgB/B,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,MAAc;QAFd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,yBAAyB;QACvB,OAAO,IAAI,CAAC,cAAc,CAAC,0BAA0B,CAAC,CAAC;IACzD,CAAC;IAED,0BAA0B;QACxB,OAAO,IAAI,CAAC,cAAc,CAAC,4BAA4B,CAAC,CAAC;IAC3D,CAAC;IAED,uBAAuB;QACrB,OAAO,IAAI,CAAC,cAAc,CAAC,wBAAwB,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,mBAAmB;QACvB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,cAAc,CAAC,IAAY;QACvC,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;SACH;QACD,OAAO,QAAQ,IAAI,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,IAAA,+BAAkB,EACtC,IAAI,CAAC,mBAAmB,CAAC,mBAAmB,CAAC,CAC9C,CAAC;YACF,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,aAAa,CAAC,KAAK;oBAC1B,IAAI,EAAE,aAAa,CAAC,IAAI;iBACzB,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qDAAqD,CAAC,EAAE,CACzD,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AAtED,gDAsEC;AAOD;;;;GAIG;AACH,SAAgB,kBAAkB,CAChC,YAA+B;IAE/B,OAAO;QACL,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACpC,OAAO,YAAY,CAAC,QAAQ,CAAC,0BAA0B,CAAC,CAAC;QAC3D,CAAC;QACD,0BAA0B,EAAE,KAAK,IAAI,EAAE;YACrC,OAAO,YAAY,CAAC,QAAQ,CAAC,4BAA4B,CAAC,CAAC;QAC7D,CAAC;QACD,uBAAuB,EAAE,KAAK,IAAI,EAAE;YAClC,OAAO,YAAY,CAAC,QAAQ,CAAC,wBAAwB,CAAC,CAAC;QACzD,CAAC;KACF,CAAC;AACJ,CAAC;AAdD,gDAcC"}

2
src/analyze-action-env.test.ts
View file

@ -25,6 +25,8 @@ test("analyze action with RAM & threads from environment variables", async (t) =
.resolves({} as actionsUtil.StatusReportBase);
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
sinon.stub(configUtils, "getConfig").resolves({
// Use GHES so we don't try to call the feature flags API endpoint
gitHubVersion: { type: util.GitHubVariant.GHES, version: "3.0.0" },
languages: [],
} as unknown as configUtils.Config);
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");

2
src/analyze-action-input.test.ts
View file

@ -25,6 +25,8 @@ test("analyze action with RAM & threads from action inputs", async (t) => {
.resolves({} as actionsUtil.StatusReportBase);
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
sinon.stub(configUtils, "getConfig").resolves({
// Use GHES so we don't try to call the feature flags API endpoint
gitHubVersion: { type: util.GitHubVariant.GHES, version: "3.0.0" },
languages: [],
} as unknown as configUtils.Config);
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");

18
src/analyze-action.ts
View file

@ -15,6 +15,7 @@ import {
import { CODEQL_VERSION_NEW_TRACING, getCodeQL } from "./codeql";
import { Config, getConfig } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { GitHubFeatureFlags } from "./feature-flags";
import { getActionsLogger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as upload_lib from "./upload-lib";
@ -99,6 +100,14 @@ async function run() {
const memory = util.getMemoryFlag(
actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
);
const featureFlags = new GitHubFeatureFlags(
config.gitHubVersion,
apiDetails,
logger
);
void featureFlags.preloadFeatureFlags();
await runFinalize(outputDir, threads, memory, config, logger);
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
runStats = await runQueries(
@ -176,7 +185,14 @@ async function run() {
const repositoryNwo = parseRepositoryNwo(
util.getRequiredEnvParam("GITHUB_REPOSITORY")
);
await uploadDatabases(repositoryNwo, config, apiDetails, logger); // Possibly upload the database bundles for remote queries
// Possibly upload the database bundles for remote queries
await uploadDatabases(
repositoryNwo,
config,
featureFlags,
apiDetails,
logger
);
if (
uploadResult !== undefined &&

107
src/database-upload.test.ts
View file

@ -10,6 +10,7 @@ import * as apiClient from "./api-client";
import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { uploadDatabases } from "./database-upload";
import { createFeatureFlags, FeatureFlags } from "./feature-flags";
import { Language } from "./languages";
import { RepositoryNwo } from "./repository";
import {
@ -32,6 +33,11 @@ test.beforeEach(() => {
initializeEnvironment(Mode.actions, "1.2.3");
});
const uploadToUploadsDomainFlags = createFeatureFlags([
"database_uploads_enabled",
"uploads_domain_enabled",
]);
const testRepoName: RepositoryNwo = { owner: "github", repo: "example" };
const testApiDetails: GitHubApiDetails = {
auth: "1234",
@ -55,44 +61,25 @@ function getTestConfig(tmpDir: string): Config {
};
}
function mockHttpRequests(
optInStatusCode: number,
useUploadDomain?: boolean,
databaseUploadStatusCode?: number
async function mockHttpRequests(
featureFlags: FeatureFlags,
databaseUploadStatusCode: number
) {
// Passing an auth token is required, so we just use a dummy value
const client = github.getOctokit("123");
const requestSpy = sinon.stub(client, "request");
const optInSpy = requestSpy.withArgs(
"GET /repos/:owner/:repo/code-scanning/codeql/databases"
);
if (optInStatusCode < 300) {
optInSpy.resolves({
status: optInStatusCode,
data: {
useUploadDomain,
},
headers: {},
url: "GET /repos/:owner/:repo/code-scanning/codeql/databases",
});
const url = (await featureFlags.getUploadsDomainEnabled())
? "POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name"
: "PUT /repos/:owner/:repo/code-scanning/codeql/databases/:language";
const databaseUploadSpy = requestSpy.withArgs(url);
if (databaseUploadStatusCode < 300) {
databaseUploadSpy.resolves(undefined);
} else {
optInSpy.throws(new HTTPError("some error message", optInStatusCode));
}
if (databaseUploadStatusCode !== undefined) {
const url = useUploadDomain
? "POST https://uploads.github.com/repos/:owner/:repo/code-scanning/codeql/databases/:language?name=:name"
: "PUT /repos/:owner/:repo/code-scanning/codeql/databases/:language";
const databaseUploadSpy = requestSpy.withArgs(url);
if (databaseUploadStatusCode < 300) {
databaseUploadSpy.resolves(undefined);
} else {
databaseUploadSpy.throws(
new HTTPError("some error message", databaseUploadStatusCode)
);
}
databaseUploadSpy.throws(
new HTTPError("some error message", databaseUploadStatusCode)
);
}
sinon.stub(apiClient, "getApiClient").value(() => client);
@ -111,6 +98,7 @@ test("Abort database upload if 'upload-database' input set to false", async (t)
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
uploadToUploadsDomainFlags,
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -140,6 +128,7 @@ test("Abort database upload if running against GHES", async (t) => {
await uploadDatabases(
testRepoName,
config,
createFeatureFlags([]),
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -169,6 +158,7 @@ test("Abort database upload if running against GHAE", async (t) => {
await uploadDatabases(
testRepoName,
config,
createFeatureFlags([]),
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -195,6 +185,7 @@ test("Abort database upload if not analyzing default branch", async (t) => {
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
uploadToUploadsDomainFlags,
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -208,7 +199,7 @@ test("Abort database upload if not analyzing default branch", async (t) => {
});
});
test("Abort database upload if opt-in request returns 404", async (t) => {
test("Abort database upload if feature flag is disabled", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
@ -217,8 +208,6 @@ test("Abort database upload if opt-in request returns 404", async (t) => {
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(404);
setCodeQL({
async databaseBundle() {
return;
@ -229,6 +218,7 @@ test("Abort database upload if opt-in request returns 404", async (t) => {
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
createFeatureFlags(["uploads_domain_enabled"]),
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -243,41 +233,6 @@ test("Abort database upload if opt-in request returns 404", async (t) => {
});
});
test("Abort database upload if opt-in request fails with something other than 404", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
sinon
.stub(actionsUtil, "getRequiredInput")
.withArgs("upload-database")
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(500);
setCodeQL({
async databaseBundle() {
return;
},
});
const loggedMessages = [] as LoggedMessage[];
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
testApiDetails,
getRecordingLogger(loggedMessages)
);
t.assert(
loggedMessages.find(
(v) =>
v.type === "info" &&
v.message ===
"Skipping database upload due to unknown error: Error: some error message"
) !== undefined
);
});
});
test("Don't crash if uploading a database fails", async (t) => {
await withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@ -287,7 +242,9 @@ test("Don't crash if uploading a database fails", async (t) => {
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, false, 500);
const featureFlags = createFeatureFlags(["database_uploads_enabled"]);
await mockHttpRequests(featureFlags, 500);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
@ -299,9 +256,13 @@ test("Don't crash if uploading a database fails", async (t) => {
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
featureFlags,
testApiDetails,
getRecordingLogger(loggedMessages)
);
console.log(loggedMessages);
t.assert(
loggedMessages.find(
(v) =>
@ -322,7 +283,7 @@ test("Successfully uploading a database to api.github.com", async (t) => {
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, false, 201);
await mockHttpRequests(uploadToUploadsDomainFlags, 201);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
@ -334,6 +295,7 @@ test("Successfully uploading a database to api.github.com", async (t) => {
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
uploadToUploadsDomainFlags,
testApiDetails,
getRecordingLogger(loggedMessages)
);
@ -356,7 +318,7 @@ test("Successfully uploading a database to uploads.github.com", async (t) => {
.returns("true");
sinon.stub(actionsUtil, "isAnalyzingDefaultBranch").resolves(true);
mockHttpRequests(200, true, 201);
await mockHttpRequests(uploadToUploadsDomainFlags, 201);
setCodeQL({
async databaseBundle(_: string, outputFilePath: string) {
@ -368,6 +330,7 @@ test("Successfully uploading a database to uploads.github.com", async (t) => {
await uploadDatabases(
testRepoName,
getTestConfig(tmpDir),
uploadToUploadsDomainFlags,
testApiDetails,
getRecordingLogger(loggedMessages)
);

27
src/database-upload.ts
View file

@ -4,6 +4,7 @@ import * as actionsUtil from "./actions-util";
import { getApiClient, GitHubApiDetails } from "./api-client";
import { getCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { FeatureFlags } from "./feature-flags";
import { Logger } from "./logging";
import { RepositoryNwo } from "./repository";
import * as util from "./util";
@ -12,6 +13,7 @@ import { bundleDb } from "./util";
export async function uploadDatabases(
repositoryNwo: RepositoryNwo,
config: Config,
featureFlags: FeatureFlags,
apiDetails: GitHubApiDetails,
logger: Logger
): Promise<void> {
@ -32,30 +34,17 @@ export async function uploadDatabases(
return;
}
const client = getApiClient(apiDetails);
let useUploadDomain: boolean;
try {
const response = await client.request(
"GET /repos/:owner/:repo/code-scanning/codeql/databases",
{
owner: repositoryNwo.owner,
repo: repositoryNwo.repo,
}
if (!(await featureFlags.getDatabaseUploadsEnabled())) {
logger.debug(
"Repository is not opted in to database uploads. Skipping upload."
);
useUploadDomain = response.data["uploads_domain_enabled"];
} catch (e) {
if (util.isHTTPError(e) && e.status === 404) {
logger.debug(
"Repository is not opted in to database uploads. Skipping upload."
);
} else {
console.log(e);
logger.info(`Skipping database upload due to unknown error: ${e}`);
}
return;
}
const client = getApiClient(apiDetails);
const codeql = await getCodeQL(config.codeQLCmd);
const useUploadDomain = await featureFlags.getUploadsDomainEnabled();
for (const language of config.languages) {
// Upload the database bundle.
// Although we are uploading arbitrary file contents to the API, it's worth

28
src/feature-flags.ts
View file

@ -3,7 +3,7 @@ import { Logger } from "./logging";
import { parseRepositoryNwo } from "./repository";
import * as util from "./util";
interface FeatureFlags {
export interface FeatureFlags {
getDatabaseUploadsEnabled(): Promise<boolean>;
getMlPoweredQueriesEnabled(): Promise<boolean>;
getUploadsDomainEnabled(): Promise<boolean>;
@ -88,3 +88,29 @@ export class GitHubFeatureFlags implements FeatureFlags {
return apiResponse;
}
}
type FeatureFlagName =
| "database_uploads_enabled"
| "ml_powered_queries_enabled"
| "uploads_domain_enabled";
/**
* Create a feature flags instance with the specified set of enabled flags.
*
* This should be only used within tests.
*/
export function createFeatureFlags(
enabledFlags: FeatureFlagName[]
): FeatureFlags {
return {
getDatabaseUploadsEnabled: async () => {
return enabledFlags.includes("database_uploads_enabled");
},
getMlPoweredQueriesEnabled: async () => {
return enabledFlags.includes("ml_powered_queries_enabled");
},
getUploadsDomainEnabled: async () => {
return enabledFlags.includes("uploads_domain_enabled");
},
};
}