robojerk/codeql-action

Merge pull request #2819 from github/update-v3.28.13-e0ea14102

Browse source 
Merge main into releases/v3
This commit is contained in:
Chuan-kai Lin 2025-03-24 06:42:41 -07:00 • committed by GitHub
commit 1b549b9259
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
26 changed files with 818 additions and 79 deletions
Download patch file Download diff file Expand all files Collapse all files

4
CHANGELOG.md
View file

@ -2,6 +2,10 @@
See the [releases page](https://github.com/github/codeql-action/releases) for the relevant changes to the CodeQL CLI and language packs.
## 3.28.13 - 24 Mar 2025
No user facing changes.
## 3.28.12 - 19 Mar 2025
- Dependency caching should now cache more dependencies for Java `build-mode: none` extractions. This should speed up workflows and avoid inconsistent alerts in some cases.

15
justfile
View file

@ -3,7 +3,7 @@ all: lint sync
# Lint source typescript
lint:
npm run lint -- --fix
npm run lint-fix
# Sync generated files (javascript and PR checks)
sync: build update-pr-checks
@ -15,3 +15,16 @@ update-pr-checks:
# Transpile typescript code into javascript
build:
npm run build
# Build then run all the tests
test: build
npm run test
# Run the tests for a single file
test_file filename: build
npx ava --verbose {{filename}}
[doc("Refresh the .js build artefacts in the lib directory")]
[confirm]
refresh-lib:
rm -rf lib && npm run build

5
lib/analyze-action.js generated
View file

@ -41,7 +41,6 @@ const fs = __importStar(require("fs"));
const path_1 = __importDefault(require("path"));
const perf_hooks_1 = require("perf_hooks");
const core = __importStar(require("@actions/core"));
const github = __importStar(require("@actions/github"));
const actionsUtil = __importStar(require("./actions-util"));
const analyze_1 = require("./analyze");
const api_client_1 = require("./api-client");
@ -194,9 +193,7 @@ async function run() {
util.checkActionVersion(actionsUtil.getActionVersion(), gitHubVersion);
const features = new feature_flags_1.Features(gitHubVersion, repositoryNwo, actionsUtil.getTemporaryDirectory(), logger);
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"], logger);
const pull_request = github.context.payload.pull_request;
const diffRangePackDir = pull_request &&
(await (0, analyze_1.setupDiffInformedQueryRun)(pull_request.base.ref, pull_request.head.label, codeql, logger, features));
const diffRangePackDir = await (0, analyze_1.setupDiffInformedQueryRun)(codeql, logger, features);
await (0, analyze_1.warnIfGoInstalledAfterInit)(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);
dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, codeql, config, logger);

2
lib/analyze-action.js.map
View file

File diff suppressed because one or more lines are too long

63
lib/analyze.js generated
View file

@ -47,6 +47,7 @@ exports.runCleanup = runCleanup;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks");
const github = __importStar(require("@actions/github"));
const io = __importStar(require("@actions/io"));
const del_1 = __importDefault(require("del"));
const yaml = __importStar(require("js-yaml"));
@ -158,24 +159,50 @@ async function finalizeDatabaseCreation(codeql, config, threadsFlag, memoryFlag,
trap_import_duration_ms: Math.round(trapImportTime),
};
}
function getPullRequestBranches() {
const pullRequest = github.context.payload.pull_request;
if (pullRequest) {
return {
base: pullRequest.base.ref,
// We use the head label instead of the head ref here, because the head
// ref lacks owner information and by itself does not uniquely identify
// the head branch (which may be in a forked repository).
head: pullRequest.head.label,
};
}
// PR analysis under Default Setup does not have the pull_request context,
// but it should set CODE_SCANNING_REF and CODE_SCANNING_BASE_BRANCH.
const codeScanningRef = process.env.CODE_SCANNING_REF;
const codeScanningBaseBranch = process.env.CODE_SCANNING_BASE_BRANCH;
if (codeScanningRef && codeScanningBaseBranch) {
return {
base: codeScanningBaseBranch,
// PR analysis under Default Setup analyzes the PR head commit instead of
// the merge commit, so we can use the provided ref directly.
head: codeScanningRef,
};
}
return undefined;
}
/**
* Set up the diff-informed analysis feature.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param codeql
* @param logger
* @param features
* @returns Absolute path to the directory containing the extension pack for
* the diff range information, or `undefined` if the feature is disabled.
*/
async function setupDiffInformedQueryRun(baseRef, headLabel, codeql, logger, features) {
async function setupDiffInformedQueryRun(codeql, logger, features) {
if (!(await features.getValue(feature_flags_1.Feature.DiffInformedQueries, codeql))) {
return undefined;
}
const branches = getPullRequestBranches();
if (!branches) {
logger.info("Not performing diff-informed analysis " +
"because we are not analyzing a pull request.");
return undefined;
}
return await (0, logging_1.withGroupAsync)("Generating diff range extension pack", async () => {
const diffRanges = await getPullRequestEditedDiffRanges(baseRef, headLabel, logger);
logger.info(`Calculating diff ranges for ${branches.base}...${branches.head}`);
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
const packDir = writeDiffRangeDataExtensionPack(logger, diffRanges);
if (packDir === undefined) {
logger.warning("Cannot create diff range extension pack for diff-informed queries; " +
@ -190,17 +217,15 @@ async function setupDiffInformedQueryRun(baseRef, headLabel, codeql, logger, fea
/**
* Return the file line ranges that were added or modified in the pull request.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param branches The base and head branches of the pull request.
* @param logger
* @returns An array of tuples, where each tuple contains the absolute path of a
* file, the start line and the end line (both 1-based and inclusive) of an
* added or modified range in that file. Returns `undefined` if the action was
* not triggered by a pull request or if there was an error.
*/
async function getPullRequestEditedDiffRanges(baseRef, headLabel, logger) {
const fileDiffs = await getFileDiffsWithBasehead(baseRef, headLabel, logger);
async function getPullRequestEditedDiffRanges(branches, logger) {
const fileDiffs = await getFileDiffsWithBasehead(branches, logger);
if (fileDiffs === undefined) {
return undefined;
}
@ -223,11 +248,11 @@ async function getPullRequestEditedDiffRanges(baseRef, headLabel, logger) {
}
return results;
}
async function getFileDiffsWithBasehead(baseRef, headLabel, logger) {
async function getFileDiffsWithBasehead(branches, logger) {
const ownerRepo = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
const owner = ownerRepo[0];
const repo = ownerRepo[1];
const basehead = `${baseRef}...${headLabel}`;
const basehead = `${branches.base}...${branches.head}`;
try {
const response = await (0, api_client_1.getApiClient)().rest.repos.compareCommitsWithBasehead({
owner,
@ -343,6 +368,14 @@ function writeDiffRangeDataExtensionPack(logger, ranges) {
if (ranges === undefined) {
return undefined;
}
if (ranges.length === 0) {
// An empty diff range means that there are no added or modified lines in
// the pull request. But the `restrictAlertsTo` extensible predicate
// interprets an empty data extension differently, as an indication that
// all alerts should be included. So we need to specifically set the diff
// range to a non-empty list that cannot match any alert location.
ranges = [{ path: "", startLine: 0, endLine: 0 }];
}
const diffRangeDir = path.join(actionsUtil.getTemporaryDirectory(), "pr-diff-range");
fs.mkdirSync(diffRangeDir);
fs.writeFileSync(path.join(diffRangeDir, "qlpack.yml"), `

2
lib/analyze.js.map
View file

File diff suppressed because one or more lines are too long

6
lib/codeql.js generated
View file

@ -292,7 +292,8 @@ async function getCodeQLForCmd(cmd, checkVersion) {
? "--force-overwrite"
: "--overwrite";
if (overlayDatabaseMode === overlay_database_utils_1.OverlayDatabaseMode.Overlay) {
extraArgs.push("--overlay");
const overlayChangesFile = await (0, overlay_database_utils_1.writeOverlayChangesFile)(config, sourceRoot, logger);
extraArgs.push(`--overlay-changes=${overlayChangesFile}`);
}
else if (overlayDatabaseMode === overlay_database_utils_1.OverlayDatabaseMode.OverlayBase) {
extraArgs.push("--overlay-base");
@ -314,6 +315,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
ignoringOptions: ["--overwrite"],
}),
], { stdin: externalRepositoryToken });
if (overlayDatabaseMode === overlay_database_utils_1.OverlayDatabaseMode.OverlayBase) {
await (0, overlay_database_utils_1.writeBaseDatabaseOidsFile)(config, sourceRoot);
}
},
async runAutobuild(config, language) {
applyAutobuildAzurePipelinesTimeoutFix();

2
lib/codeql.js.map
View file

File diff suppressed because one or more lines are too long

56
lib/git-utils.js generated
View file

@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.getGitRoot = exports.decodeGitFilePath = exports.gitRepack = exports.gitFetch = exports.deepenGitHistory = exports.determineBaseBranchHeadCommitOid = exports.getCommitOid = void 0;
exports.getFileOidsUnderPath = exports.getGitRoot = exports.decodeGitFilePath = exports.gitRepack = exports.gitFetch = exports.deepenGitHistory = exports.determineBaseBranchHeadCommitOid = exports.getCommitOid = exports.runGitCommand = void 0;
exports.getRef = getRef;
exports.isAnalyzingDefaultBranch = isAnalyzingDefaultBranch;
const core = __importStar(require("@actions/core"));
@ -41,7 +41,7 @@ const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
const io = __importStar(require("@actions/io"));
const actions_util_1 = require("./actions-util");
const util_1 = require("./util");
async function runGitCommand(checkoutPath, args, customErrorMessage) {
const runGitCommand = async function (workingDirectory, args, customErrorMessage) {
let stdout = "";
let stderr = "";
core.debug(`Running git command: git ${args.join(" ")}`);
@ -56,7 +56,7 @@ async function runGitCommand(checkoutPath, args, customErrorMessage) {
stderr += data.toString();
},
},
cwd: checkoutPath,
cwd: workingDirectory,
}).exec();
return stdout;
}
@ -69,7 +69,8 @@ async function runGitCommand(checkoutPath, args, customErrorMessage) {
core.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
throw error;
}
}
};
exports.runGitCommand = runGitCommand;
/**
* Gets the SHA of the commit that is currently checked out.
*/
@ -82,7 +83,7 @@ const getCommitOid = async function (checkoutPath, ref = "HEAD") {
// Even if this does go wrong, it's not a huge problem for the alerts to
// reported on the merge commit.
try {
const stdout = await runGitCommand(checkoutPath, ["rev-parse", ref], "Continuing with commit SHA from user input or environment.");
const stdout = await (0, exports.runGitCommand)(checkoutPath, ["rev-parse", ref], "Continuing with commit SHA from user input or environment.");
return stdout.trim();
}
catch {
@ -106,7 +107,7 @@ const determineBaseBranchHeadCommitOid = async function (checkoutPathOverride) {
let commitOid = "";
let baseOid = "";
let headOid = "";
const stdout = await runGitCommand(checkoutPath, ["show", "-s", "--format=raw", mergeSha], "Will calculate the base branch SHA on the server.");
const stdout = await (0, exports.runGitCommand)(checkoutPath, ["show", "-s", "--format=raw", mergeSha], "Will calculate the base branch SHA on the server.");
for (const data of stdout.split("\n")) {
if (data.startsWith("commit ") && commitOid === "") {
commitOid = data.substring(7);
@ -141,7 +142,7 @@ exports.determineBaseBranchHeadCommitOid = determineBaseBranchHeadCommitOid;
*/
const deepenGitHistory = async function () {
try {
await runGitCommand((0, actions_util_1.getOptionalInput)("checkout_path"), [
await (0, exports.runGitCommand)((0, actions_util_1.getOptionalInput)("checkout_path"), [
"fetch",
"origin",
"HEAD",
@ -163,7 +164,7 @@ exports.deepenGitHistory = deepenGitHistory;
*/
const gitFetch = async function (branch, extraFlags) {
try {
await runGitCommand((0, actions_util_1.getOptionalInput)("checkout_path"), ["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`], `Cannot fetch ${branch}.`);
await (0, exports.runGitCommand)((0, actions_util_1.getOptionalInput)("checkout_path"), ["fetch", "--no-tags", ...extraFlags, "origin", `${branch}:${branch}`], `Cannot fetch ${branch}.`);
}
catch {
// Errors are already logged by runGitCommand()
@ -178,7 +179,7 @@ exports.gitFetch = gitFetch;
*/
const gitRepack = async function (flags) {
try {
await runGitCommand((0, actions_util_1.getOptionalInput)("checkout_path"), ["repack", ...flags], "Cannot repack the repository.");
await (0, exports.runGitCommand)((0, actions_util_1.getOptionalInput)("checkout_path"), ["repack", ...flags], "Cannot repack the repository.");
}
catch {
// Errors are already logged by runGitCommand()
@ -238,7 +239,7 @@ exports.decodeGitFilePath = decodeGitFilePath;
*/
const getGitRoot = async function (sourceRoot) {
try {
const stdout = await runGitCommand(sourceRoot, ["rev-parse", "--show-toplevel"], `Cannot find Git repository root from the source root ${sourceRoot}.`);
const stdout = await (0, exports.runGitCommand)(sourceRoot, ["rev-parse", "--show-toplevel"], `Cannot find Git repository root from the source root ${sourceRoot}.`);
return stdout.trim();
}
catch {
@ -247,6 +248,41 @@ const getGitRoot = async function (sourceRoot) {
}
};
exports.getGitRoot = getGitRoot;
/**
* Returns the Git OIDs of all tracked files (in the index and in the working
* tree) that are under the given base path, including files in active
* submodules. Untracked files and files not under the given base path are
* ignored.
*
* @param basePath A path into the Git repository.
* @returns a map from file paths (relative to `basePath`) to Git OIDs.
* @throws {Error} if "git ls-tree" produces unexpected output.
*/
const getFileOidsUnderPath = async function (basePath) {
// Without the --full-name flag, the path is relative to the current working
// directory of the git command, which is basePath.
const stdout = await (0, exports.runGitCommand)(basePath, ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"], "Cannot list Git OIDs of tracked files.");
const fileOidMap = {};
// With --format=%(objectname)_%(path), the output is a list of lines like:
// 30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js
// d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map
const regex = /^([0-9a-f]{40})_(.+)$/;
for (const line of stdout.split("\n")) {
if (line) {
const match = line.match(regex);
if (match) {
const oid = match[1];
const path = (0, exports.decodeGitFilePath)(match[2]);
fileOidMap[path] = oid;
}
else {
throw new Error(`Unexpected "git ls-files" output: ${line}`);
}
}
}
return fileOidMap;
};
exports.getFileOidsUnderPath = getFileOidsUnderPath;
function getRefFromEnv() {
// To workaround a limitation of Actions dynamic workflows not setting
// the GITHUB_REF in some cases, we accept also the ref within the

2
lib/git-utils.js.map
View file

File diff suppressed because one or more lines are too long

71
lib/git-utils.test.js generated
View file

@ -265,4 +265,75 @@ const util_1 = require("./util");
t.deepEqual(gitUtils.decodeGitFilePath('"foo\\vbar"'), "foo\vbar");
t.deepEqual(gitUtils.decodeGitFilePath('"\\a\\b\\f\\n\\r\\t\\v"'), "\x07\b\f\n\r\t\v");
});
(0, ava_1.default)("getFileOidsUnderPath returns correct file mapping", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils, "runGitCommand")
.resolves("30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts");
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
t.deepEqual(runGitCommandStub.firstCall.args, [
"/fake/path",
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
]);
}
finally {
runGitCommandStub.restore();
}
});
(0, ava_1.default)("getFileOidsUnderPath handles quoted paths", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils, "runGitCommand")
.resolves("30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
'd89514599a9a99f22b4085766d40af7b99974827_"lib/file with spaces.js"\n' +
'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"');
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
}
finally {
runGitCommandStub.restore();
}
});
(0, ava_1.default)("getFileOidsUnderPath handles empty output", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils, "runGitCommand")
.resolves("");
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {});
}
finally {
runGitCommandStub.restore();
}
});
(0, ava_1.default)("getFileOidsUnderPath throws on unexpected output format", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils, "runGitCommand")
.resolves("30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"invalid-line-format\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts");
try {
await t.throwsAsync(async () => {
await gitUtils.getFileOidsUnderPath("/fake/path");
}, {
instanceOf: Error,
message: 'Unexpected "git ls-files" output: invalid-line-format',
});
}
finally {
runGitCommandStub.restore();
}
});
//# sourceMappingURL=git-utils.test.js.map

2
lib/git-utils.test.js.map
View file

File diff suppressed because one or more lines are too long

118
lib/overlay-database-utils.js generated
View file

@ -1,6 +1,45 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.CODEQL_OVERLAY_MINIMUM_VERSION = exports.OverlayDatabaseMode = void 0;
exports.writeBaseDatabaseOidsFile = writeBaseDatabaseOidsFile;
exports.writeOverlayChangesFile = writeOverlayChangesFile;
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const actions_util_1 = require("./actions-util");
const git_utils_1 = require("./git-utils");
var OverlayDatabaseMode;
(function (OverlayDatabaseMode) {
OverlayDatabaseMode["Overlay"] = "overlay";
@ -8,4 +47,83 @@ var OverlayDatabaseMode;
OverlayDatabaseMode["None"] = "none";
})(OverlayDatabaseMode || (exports.OverlayDatabaseMode = OverlayDatabaseMode = {}));
exports.CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
/**
* Writes a JSON file containing Git OIDs for all tracked files (represented
* by path relative to the source root) under the source root. The file is
* written into the database location specified in the config.
*
* @param config The configuration object containing the database location
* @param sourceRoot The root directory containing the source files to process
* @throws {Error} If the Git repository root cannot be determined
*/
async function writeBaseDatabaseOidsFile(config, sourceRoot) {
const gitFileOids = await (0, git_utils_1.getFileOidsUnderPath)(sourceRoot);
const gitFileOidsJson = JSON.stringify(gitFileOids);
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
await fs.promises.writeFile(baseDatabaseOidsFilePath, gitFileOidsJson);
}
/**
* Reads and parses the JSON file containing the base database Git OIDs.
* This file contains the mapping of file paths to their corresponding Git OIDs
* that was previously written by writeBaseDatabaseOidsFile().
*
* @param config The configuration object containing the database location
* @param logger The logger instance to use for error reporting
* @returns An object mapping file paths (relative to source root) to their Git OIDs
* @throws {Error} If the file cannot be read or parsed
*/
async function readBaseDatabaseOidsFile(config, logger) {
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
try {
const contents = await fs.promises.readFile(baseDatabaseOidsFilePath, "utf-8");
return JSON.parse(contents);
}
catch (e) {
logger.error("Failed to read overlay-base file OIDs from " +
`${baseDatabaseOidsFilePath}: ${e.message || e}`);
throw e;
}
}
function getBaseDatabaseOidsFilePath(config) {
return path.join(config.dbLocation, "base-database-oids.json");
}
/**
* Writes a JSON file containing the source-root-relative paths of files under
* `sourceRoot` that have changed (added, removed, or modified) from the overlay
* base database.
*
* This function uses the Git index to determine which files have changed, so it
* requires the following preconditions, both when this function is called and
* when the overlay-base database was initialized:
*
* - It requires that `sourceRoot` is inside a Git repository.
* - It assumes that all changes in the working tree are staged in the index.
* - It assumes that all files of interest are tracked by Git, e.g. not covered
* by `.gitignore`.
*/
async function writeOverlayChangesFile(config, sourceRoot, logger) {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await (0, git_utils_1.getFileOidsUnderPath)(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`);
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path.join((0, actions_util_1.getTemporaryDirectory)(), "overlay-changes.json");
logger.debug(`Writing overlay changed files to ${overlayChangesFile}: ${changedFilesJson}`);
await fs.promises.writeFile(overlayChangesFile, changedFilesJson);
return overlayChangesFile;
}
function computeChangedFiles(baseFileOids, overlayFileOids) {
const changes = [];
for (const [file, oid] of Object.entries(overlayFileOids)) {
if (!(file in baseFileOids) || baseFileOids[file] !== oid) {
changes.push(file);
}
}
for (const file of Object.keys(baseFileOids)) {
if (!(file in overlayFileOids)) {
changes.push(file);
}
}
return changes;
}
//# sourceMappingURL=overlay-database-utils.js.map

2
lib/overlay-database-utils.js.map
View file

@ -1 +1 @@
{"version":3,"file":"overlay-database-utils.js","sourceRoot":"","sources":["../src/overlay-database-utils.ts"],"names":[],"mappings":";;;AAAA,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,0CAAmB,CAAA;IACnB,mDAA4B,CAAA;IAC5B,oCAAa,CAAA;AACf,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B;AAEY,QAAA,8BAA8B,GAAG,QAAQ,CAAC"}
{"version":3,"file":"overlay-database-utils.js","sourceRoot":"","sources":["../src/overlay-database-utils.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyBA,8DAQC;AAkDD,0DAsBC;AAzGD,uCAAyB;AACzB,2CAA6B;AAE7B,iDAAuD;AAEvD,2CAAmD;AAGnD,IAAY,mBAIX;AAJD,WAAY,mBAAmB;IAC7B,0CAAmB,CAAA;IACnB,mDAA4B,CAAA;IAC5B,oCAAa,CAAA;AACf,CAAC,EAJW,mBAAmB,mCAAnB,mBAAmB,QAI9B;AAEY,QAAA,8BAA8B,GAAG,QAAQ,CAAC;AAEvD;;;;;;;;GAQG;AACI,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,UAAkB;IAElB,MAAM,WAAW,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;IACpD,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,wBAAwB,EAAE,eAAe,CAAC,CAAC;AACzE,CAAC;AAED;;;;;;;;;GASG;AACH,KAAK,UAAU,wBAAwB,CACrC,MAAc,EACd,MAAc;IAEd,MAAM,wBAAwB,GAAG,2BAA2B,CAAC,MAAM,CAAC,CAAC;IACrE,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CACzC,wBAAwB,EACxB,OAAO,CACR,CAAC;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAA8B,CAAC;IAC3D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CACV,6CAA6C;YAC3C,GAAG,wBAAwB,KAAM,CAAS,CAAC,OAAO,IAAI,CAAC,EAAE,CAC5D,CAAC;QACF,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAAC,MAAc;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;AACjE,CAAC;AAED;;;;;;;;;;;;;GAaG;AACI,KAAK,UAAU,uBAAuB,CAC3C,MAAc,EACd,UAAkB,EAClB,MAAc;IAEd,MAAM,YAAY,GAAG,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpE,MAAM,eAAe,GAAG,MAAM,IAAA,gCAAoB,EAAC,UAAU,CAAC,CAAC;IAC/D,MAAM,YAAY,GAAG,mBAAmB,CAAC,YAAY,EAAE,eAAe,CAAC,CAAC;IACxE,MAAM,CAAC,IAAI,CACT,SAAS,YAAY,CAAC,MAAM,0BAA0B,UAAU,GAAG,CACpE,CAAC;IAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC,CAAC;IACnE,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAClC,IAAA,oCAAqB,GAAE,EACvB,sBAAsB,CACvB,CAAC;IACF,MAAM,CAAC,KAAK,CACV,oCAAoC,kBAAkB,KAAK,gBAAgB,EAAE,CAC9E,CAAC;IACF,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,kBAAkB,EAAE,gBAAgB,CAAC,CAAC;IAClE,OAAO,kBAAkB,CAAC;AAC5B,CAAC;AAED,SAAS,mBAAmB,CAC1B,YAAuC,EACvC,eAA0C;IAE1C,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,eAAe,CAAC,EAAE,CAAC;QAC1D,IAAI,CAAC,CAAC,IAAI,IAAI,YAAY,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7C,IAAI,CAAC,CAAC,IAAI,IAAI,eAAe,CAAC,EAAE,CAAC;YAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

94
lib/overlay-database-utils.test.js generated Normal file
View file

@ -0,0 +1,94 @@
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const ava_1 = __importDefault(require("ava"));
const sinon = __importStar(require("sinon"));
const actionsUtil = __importStar(require("./actions-util"));
const gitUtils = __importStar(require("./git-utils"));
const logging_1 = require("./logging");
const overlay_database_utils_1 = require("./overlay-database-utils");
const testing_utils_1 = require("./testing-utils");
const util_1 = require("./util");
(0, testing_utils_1.setupTests)(ava_1.default);
(0, ava_1.default)("writeOverlayChangesFile generates correct changes file", async (t) => {
await (0, util_1.withTmpDir)(async (tmpDir) => {
const dbLocation = path.join(tmpDir, "db");
await fs.promises.mkdir(dbLocation, { recursive: true });
const sourceRoot = path.join(tmpDir, "src");
await fs.promises.mkdir(sourceRoot, { recursive: true });
const tempDir = path.join(tmpDir, "temp");
await fs.promises.mkdir(tempDir, { recursive: true });
const logger = (0, logging_1.getRunnerLogger)(true);
const config = (0, testing_utils_1.createTestConfig)({ dbLocation });
// Mock the getFileOidsUnderPath function to return base OIDs
const baseOids = {
"unchanged.js": "aaa111",
"modified.js": "bbb222",
"deleted.js": "ccc333",
};
const getFileOidsStubForBase = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(baseOids);
// Write the base database OIDs file
await (0, overlay_database_utils_1.writeBaseDatabaseOidsFile)(config, sourceRoot);
getFileOidsStubForBase.restore();
// Mock the getFileOidsUnderPath function to return overlay OIDs
const currentOids = {
"unchanged.js": "aaa111",
"modified.js": "ddd444", // Changed OID
"added.js": "eee555", // New file
};
const getFileOidsStubForOverlay = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(currentOids);
// Write the overlay changes file, which uses the mocked overlay OIDs
// and the base database OIDs file
const getTempDirStub = sinon
.stub(actionsUtil, "getTemporaryDirectory")
.returns(tempDir);
const changesFilePath = await (0, overlay_database_utils_1.writeOverlayChangesFile)(config, sourceRoot, logger);
getFileOidsStubForOverlay.restore();
getTempDirStub.restore();
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
const parsedContent = JSON.parse(fileContent);
t.deepEqual(parsedContent.changes.sort(), ["added.js", "deleted.js", "modified.js"], "Should identify added, deleted, and modified files");
});
});
//# sourceMappingURL=overlay-database-utils.test.js.map

1
lib/overlay-database-utils.test.js.map Normal file
View file

@ -0,0 +1 @@
{"version":3,"file":"overlay-database-utils.test.js","sourceRoot":"","sources":["../src/overlay-database-utils.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,sDAAwC;AACxC,uCAA4C;AAC5C,qEAGkC;AAClC,mDAA+D;AAC/D,iCAAoC;AAEpC,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,IAAA,aAAI,EAAC,wDAAwD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACzE,MAAM,IAAA,iBAAU,EAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAChC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC3C,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1C,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,IAAA,yBAAe,EAAC,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAA,gCAAgB,EAAC,EAAE,UAAU,EAAE,CAAC,CAAC;QAEhD,6DAA6D;QAC7D,MAAM,QAAQ,GAAG;YACf,cAAc,EAAE,QAAQ;YACxB,aAAa,EAAE,QAAQ;YACvB,YAAY,EAAE,QAAQ;SACvB,CAAC;QACF,MAAM,sBAAsB,GAAG,KAAK;aACjC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC;aACtC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEtB,oCAAoC;QACpC,MAAM,IAAA,kDAAyB,EAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACpD,sBAAsB,CAAC,OAAO,EAAE,CAAC;QAEjC,gEAAgE;QAChE,MAAM,WAAW,GAAG;YAClB,cAAc,EAAE,QAAQ;YACxB,aAAa,EAAE,QAAQ,EAAE,cAAc;YACvC,UAAU,EAAE,QAAQ,EAAE,WAAW;SAClC,CAAC;QACF,MAAM,yBAAyB,GAAG,KAAK;aACpC,IAAI,CAAC,QAAQ,EAAE,sBAAsB,CAAC;aACtC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAEzB,qEAAqE;QACrE,kCAAkC;QAClC,MAAM,cAAc,GAAG,KAAK;aACzB,IAAI,CAAC,WAAW,EAAE,uBAAuB,CAAC;aAC1C,OAAO,CAAC,OAAO,CAAC,CAAC;QACpB,MAAM,eAAe,GAAG,MAAM,IAAA,gDAAuB,EACnD,MAAM,EACN,UAAU,EACV,MAAM,CACP,CAAC;QACF,yBAAyB,CAAC,OAAO,EAAE,CAAC;QACpC,cAAc,CAAC,OAAO,EAAE,CAAC;QAEzB,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;QACzE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAA0B,CAAC;QAEvE,CAAC,CAAC,SAAS,CACT,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,EAC5B,CAAC,UAAU,EAAE,YAAY,EAAE,aAAa,CAAC,EACzC,oDAAoD,CACrD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

2
node_modules/.package-lock.json generated vendored
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "3.28.12",
"version": "3.28.13",
"lockfileVersion": 3,
"requires": true,
"packages": {

4
package-lock.json generated
View file

@ -1,12 +1,12 @@
{
"name": "codeql",
"version": "3.28.12",
"version": "3.28.13",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "codeql",
"version": "3.28.12",
"version": "3.28.13",
"license": "MIT",
"dependencies": {
"@actions/artifact": "^2.3.1",

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "codeql",
"version": "3.28.12",
"version": "3.28.13",
"private": true,
"description": "CodeQL action",
"scripts": {

16
src/analyze-action.ts
View file

@ -3,7 +3,6 @@ import path from "path";
import { performance } from "perf_hooks";
import * as core from "@actions/core";
import * as github from "@actions/github";
import * as actionsUtil from "./actions-util";
import {
@ -272,16 +271,11 @@ async function run() {
logger,
);
const pull_request = github.context.payload.pull_request;
const diffRangePackDir =
pull_request &&
(await setupDiffInformedQueryRun(
pull_request.base.ref as string,
pull_request.head.label as string,
codeql,
logger,
features,
));
const diffRangePackDir = await setupDiffInformedQueryRun(
codeql,
logger,
features,
);
await warnIfGoInstalledAfterInit(config, logger);
await runAutobuildIfLegacyGoWorkflow(config, logger);

81
src/analyze.ts
View file

@ -2,6 +2,7 @@ import * as fs from "fs";
import * as path from "path";
import { performance } from "perf_hooks";
import * as github from "@actions/github";
import * as io from "@actions/io";
import del from "del";
import * as yaml from "js-yaml";
@ -253,21 +254,45 @@ async function finalizeDatabaseCreation(
};
}
interface PullRequestBranches {
base: string;
head: string;
}
function getPullRequestBranches(): PullRequestBranches | undefined {
const pullRequest = github.context.payload.pull_request;
if (pullRequest) {
return {
base: pullRequest.base.ref,
// We use the head label instead of the head ref here, because the head
// ref lacks owner information and by itself does not uniquely identify
// the head branch (which may be in a forked repository).
head: pullRequest.head.label,
};
}
// PR analysis under Default Setup does not have the pull_request context,
// but it should set CODE_SCANNING_REF and CODE_SCANNING_BASE_BRANCH.
const codeScanningRef = process.env.CODE_SCANNING_REF;
const codeScanningBaseBranch = process.env.CODE_SCANNING_BASE_BRANCH;
if (codeScanningRef && codeScanningBaseBranch) {
return {
base: codeScanningBaseBranch,
// PR analysis under Default Setup analyzes the PR head commit instead of
// the merge commit, so we can use the provided ref directly.
head: codeScanningRef,
};
}
return undefined;
}
/**
* Set up the diff-informed analysis feature.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param codeql
* @param logger
* @param features
* @returns Absolute path to the directory containing the extension pack for
* the diff range information, or `undefined` if the feature is disabled.
*/
export async function setupDiffInformedQueryRun(
baseRef: string,
headLabel: string,
codeql: CodeQL,
logger: Logger,
features: FeatureEnablement,
@ -275,14 +300,23 @@ export async function setupDiffInformedQueryRun(
if (!(await features.getValue(Feature.DiffInformedQueries, codeql))) {
return undefined;
}
const branches = getPullRequestBranches();
if (!branches) {
logger.info(
"Not performing diff-informed analysis " +
"because we are not analyzing a pull request.",
);
return undefined;
}
return await withGroupAsync(
"Generating diff range extension pack",
async () => {
const diffRanges = await getPullRequestEditedDiffRanges(
baseRef,
headLabel,
logger,
logger.info(
`Calculating diff ranges for ${branches.base}...${branches.head}`,
);
const diffRanges = await getPullRequestEditedDiffRanges(branches, logger);
const packDir = writeDiffRangeDataExtensionPack(logger, diffRanges);
if (packDir === undefined) {
logger.warning(
@ -302,9 +336,7 @@ export async function setupDiffInformedQueryRun(
/**
* Return the file line ranges that were added or modified in the pull request.
*
* @param baseRef The base branch name, used for calculating the diff range.
* @param headLabel The label that uniquely identifies the head branch across
* repositories, used for calculating the diff range.
* @param branches The base and head branches of the pull request.
* @param logger
* @returns An array of tuples, where each tuple contains the absolute path of a
* file, the start line and the end line (both 1-based and inclusive) of an
@ -312,11 +344,10 @@ export async function setupDiffInformedQueryRun(
* not triggered by a pull request or if there was an error.
*/
async function getPullRequestEditedDiffRanges(
baseRef: string,
headLabel: string,
branches: PullRequestBranches,
logger: Logger,
): Promise<DiffThunkRange[] | undefined> {
const fileDiffs = await getFileDiffsWithBasehead(baseRef, headLabel, logger);
const fileDiffs = await getFileDiffsWithBasehead(branches, logger);
if (fileDiffs === undefined) {
return undefined;
}
@ -355,14 +386,13 @@ interface FileDiff {
}
async function getFileDiffsWithBasehead(
baseRef: string,
headLabel: string,
branches: PullRequestBranches,
logger: Logger,
): Promise<FileDiff[] | undefined> {
const ownerRepo = util.getRequiredEnvParam("GITHUB_REPOSITORY").split("/");
const owner = ownerRepo[0];
const repo = ownerRepo[1];
const basehead = `${baseRef}...${headLabel}`;
const basehead = `${branches.base}...${branches.head}`;
try {
const response = await getApiClient().rest.repos.compareCommitsWithBasehead(
{
@ -497,6 +527,15 @@ function writeDiffRangeDataExtensionPack(
return undefined;
}
if (ranges.length === 0) {
// An empty diff range means that there are no added or modified lines in
// the pull request. But the `restrictAlertsTo` extensible predicate
// interprets an empty data extension differently, as an indication that
// all alerts should be included. So we need to specifically set the diff
// range to a non-empty list that cannot match any alert location.
ranges = [{ path: "", startLine: 0, endLine: 0 }];
}
const diffRangeDir = path.join(
actionsUtil.getTemporaryDirectory(),
"pr-diff-range",

17
src/codeql.ts
View file

@ -24,7 +24,11 @@ import {
import { isAnalyzingDefaultBranch } from "./git-utils";
import { Language } from "./languages";
import { Logger } from "./logging";
import { OverlayDatabaseMode } from "./overlay-database-utils";
import {
OverlayDatabaseMode,
writeBaseDatabaseOidsFile,
writeOverlayChangesFile,
} from "./overlay-database-utils";
import * as setupCodeql from "./setup-codeql";
import { ZstdAvailability } from "./tar";
import { ToolsDownloadStatusReport } from "./tools-download";
@ -610,7 +614,12 @@ export async function getCodeQLForCmd(
: "--overwrite";
if (overlayDatabaseMode === OverlayDatabaseMode.Overlay) {
extraArgs.push("--overlay");
const overlayChangesFile = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
extraArgs.push(`--overlay-changes=${overlayChangesFile}`);
} else if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
extraArgs.push("--overlay-base");
}
@ -636,6 +645,10 @@ export async function getCodeQLForCmd(
],
{ stdin: externalRepositoryToken },
);
if (overlayDatabaseMode === OverlayDatabaseMode.OverlayBase) {
await writeBaseDatabaseOidsFile(config, sourceRoot);
}
},
async runAutobuild(config: Config, language: Language) {
applyAutobuildAzurePipelinesTimeoutFix();

87
src/git-utils.test.ts
View file

@ -305,3 +305,90 @@ test("decodeGitFilePath quoted strings", async (t) => {
"\x07\b\f\n\r\t\v",
);
});
test("getFileOidsUnderPath returns correct file mapping", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/git-utils.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/git-utils.js.map": "d89514599a9a99f22b4085766d40af7b99974827",
"src/git-utils.ts": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
t.deepEqual(runGitCommandStub.firstCall.args, [
"/fake/path",
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
]);
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath handles quoted paths", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
'd89514599a9a99f22b4085766d40af7b99974827_"lib/file with spaces.js"\n' +
'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"',
);
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {
"lib/normal-file.js": "30d998ded095371488be3a729eb61d86ed721a18",
"lib/file with spaces.js": "d89514599a9a99f22b4085766d40af7b99974827",
"lib/file\twith\ttabs.js": "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96",
});
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath handles empty output", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves("");
try {
const result = await gitUtils.getFileOidsUnderPath("/fake/path");
t.deepEqual(result, {});
} finally {
runGitCommandStub.restore();
}
});
test("getFileOidsUnderPath throws on unexpected output format", async (t) => {
const runGitCommandStub = sinon
.stub(gitUtils as any, "runGitCommand")
.resolves(
"30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
"invalid-line-format\n" +
"a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
);
try {
await t.throwsAsync(
async () => {
await gitUtils.getFileOidsUnderPath("/fake/path");
},
{
instanceOf: Error,
message: 'Unexpected "git ls-files" output: invalid-line-format',
},
);
} finally {
runGitCommandStub.restore();
}
});

49
src/git-utils.ts
View file

@ -9,8 +9,8 @@ import {
} from "./actions-util";
import { ConfigurationError, getRequiredEnvParam } from "./util";
async function runGitCommand(
checkoutPath: string | undefined,
export const runGitCommand = async function (
workingDirectory: string | undefined,
args: string[],
customErrorMessage: string,
): Promise<string> {
@ -28,7 +28,7 @@ async function runGitCommand(
stderr += data.toString();
},
},
cwd: checkoutPath,
cwd: workingDirectory,
}).exec();
return stdout;
} catch (error) {
@ -40,7 +40,7 @@ async function runGitCommand(
core.info(`git call failed. ${customErrorMessage} Error: ${reason}`);
throw error;
}
}
};
/**
* Gets the SHA of the commit that is currently checked out.
@ -253,6 +253,47 @@ export const getGitRoot = async function (
}
};
/**
* Returns the Git OIDs of all tracked files (in the index and in the working
* tree) that are under the given base path, including files in active
* submodules. Untracked files and files not under the given base path are
* ignored.
*
* @param basePath A path into the Git repository.
* @returns a map from file paths (relative to `basePath`) to Git OIDs.
* @throws {Error} if "git ls-tree" produces unexpected output.
*/
export const getFileOidsUnderPath = async function (
basePath: string,
): Promise<{ [key: string]: string }> {
// Without the --full-name flag, the path is relative to the current working
// directory of the git command, which is basePath.
const stdout = await runGitCommand(
basePath,
["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
"Cannot list Git OIDs of tracked files.",
);
const fileOidMap: { [key: string]: string } = {};
// With --format=%(objectname)_%(path), the output is a list of lines like:
// 30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js
// d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map
const regex = /^([0-9a-f]{40})_(.+)$/;
for (const line of stdout.split("\n")) {
if (line) {
const match = line.match(regex);
if (match) {
const oid = match[1];
const path = decodeGitFilePath(match[2]);
fileOidMap[path] = oid;
} else {
throw new Error(`Unexpected "git ls-files" output: ${line}`);
}
}
}
return fileOidMap;
};
function getRefFromEnv(): string {
// To workaround a limitation of Actions dynamic workflows not setting
// the GITHUB_REF in some cases, we accept also the ref within the

77
src/overlay-database-utils.test.ts Normal file
View file

@ -0,0 +1,77 @@
import * as fs from "fs";
import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import * as actionsUtil from "./actions-util";
import * as gitUtils from "./git-utils";
import { getRunnerLogger } from "./logging";
import {
writeBaseDatabaseOidsFile,
writeOverlayChangesFile,
} from "./overlay-database-utils";
import { createTestConfig, setupTests } from "./testing-utils";
import { withTmpDir } from "./util";
setupTests(test);
test("writeOverlayChangesFile generates correct changes file", async (t) => {
await withTmpDir(async (tmpDir) => {
const dbLocation = path.join(tmpDir, "db");
await fs.promises.mkdir(dbLocation, { recursive: true });
const sourceRoot = path.join(tmpDir, "src");
await fs.promises.mkdir(sourceRoot, { recursive: true });
const tempDir = path.join(tmpDir, "temp");
await fs.promises.mkdir(tempDir, { recursive: true });
const logger = getRunnerLogger(true);
const config = createTestConfig({ dbLocation });
// Mock the getFileOidsUnderPath function to return base OIDs
const baseOids = {
"unchanged.js": "aaa111",
"modified.js": "bbb222",
"deleted.js": "ccc333",
};
const getFileOidsStubForBase = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(baseOids);
// Write the base database OIDs file
await writeBaseDatabaseOidsFile(config, sourceRoot);
getFileOidsStubForBase.restore();
// Mock the getFileOidsUnderPath function to return overlay OIDs
const currentOids = {
"unchanged.js": "aaa111",
"modified.js": "ddd444", // Changed OID
"added.js": "eee555", // New file
};
const getFileOidsStubForOverlay = sinon
.stub(gitUtils, "getFileOidsUnderPath")
.resolves(currentOids);
// Write the overlay changes file, which uses the mocked overlay OIDs
// and the base database OIDs file
const getTempDirStub = sinon
.stub(actionsUtil, "getTemporaryDirectory")
.returns(tempDir);
const changesFilePath = await writeOverlayChangesFile(
config,
sourceRoot,
logger,
);
getFileOidsStubForOverlay.restore();
getTempDirStub.restore();
const fileContent = await fs.promises.readFile(changesFilePath, "utf-8");
const parsedContent = JSON.parse(fileContent) as { changes: string[] };
t.deepEqual(
parsedContent.changes.sort(),
["added.js", "deleted.js", "modified.js"],
"Should identify added, deleted, and modified files",
);
});
});

117
src/overlay-database-utils.ts
View file

@ -1,3 +1,11 @@
import * as fs from "fs";
import * as path from "path";
import { getTemporaryDirectory } from "./actions-util";
import { type Config } from "./config-utils";
import { getFileOidsUnderPath } from "./git-utils";
import { Logger } from "./logging";
export enum OverlayDatabaseMode {
Overlay = "overlay",
OverlayBase = "overlay-base",
@ -5,3 +13,112 @@ export enum OverlayDatabaseMode {
}
export const CODEQL_OVERLAY_MINIMUM_VERSION = "2.20.5";
/**
* Writes a JSON file containing Git OIDs for all tracked files (represented
* by path relative to the source root) under the source root. The file is
* written into the database location specified in the config.
*
* @param config The configuration object containing the database location
* @param sourceRoot The root directory containing the source files to process
* @throws {Error} If the Git repository root cannot be determined
*/
export async function writeBaseDatabaseOidsFile(
config: Config,
sourceRoot: string,
): Promise<void> {
const gitFileOids = await getFileOidsUnderPath(sourceRoot);
const gitFileOidsJson = JSON.stringify(gitFileOids);
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
await fs.promises.writeFile(baseDatabaseOidsFilePath, gitFileOidsJson);
}
/**
* Reads and parses the JSON file containing the base database Git OIDs.
* This file contains the mapping of file paths to their corresponding Git OIDs
* that was previously written by writeBaseDatabaseOidsFile().
*
* @param config The configuration object containing the database location
* @param logger The logger instance to use for error reporting
* @returns An object mapping file paths (relative to source root) to their Git OIDs
* @throws {Error} If the file cannot be read or parsed
*/
async function readBaseDatabaseOidsFile(
config: Config,
logger: Logger,
): Promise<{ [key: string]: string }> {
const baseDatabaseOidsFilePath = getBaseDatabaseOidsFilePath(config);
try {
const contents = await fs.promises.readFile(
baseDatabaseOidsFilePath,
"utf-8",
);
return JSON.parse(contents) as { [key: string]: string };
} catch (e) {
logger.error(
"Failed to read overlay-base file OIDs from " +
`${baseDatabaseOidsFilePath}: ${(e as any).message || e}`,
);
throw e;
}
}
function getBaseDatabaseOidsFilePath(config: Config): string {
return path.join(config.dbLocation, "base-database-oids.json");
}
/**
* Writes a JSON file containing the source-root-relative paths of files under
* `sourceRoot` that have changed (added, removed, or modified) from the overlay
* base database.
*
* This function uses the Git index to determine which files have changed, so it
* requires the following preconditions, both when this function is called and
* when the overlay-base database was initialized:
*
* - It requires that `sourceRoot` is inside a Git repository.
* - It assumes that all changes in the working tree are staged in the index.
* - It assumes that all files of interest are tracked by Git, e.g. not covered
* by `.gitignore`.
*/
export async function writeOverlayChangesFile(
config: Config,
sourceRoot: string,
logger: Logger,
): Promise<string> {
const baseFileOids = await readBaseDatabaseOidsFile(config, logger);
const overlayFileOids = await getFileOidsUnderPath(sourceRoot);
const changedFiles = computeChangedFiles(baseFileOids, overlayFileOids);
logger.info(
`Found ${changedFiles.length} changed file(s) under ${sourceRoot}.`,
);
const changedFilesJson = JSON.stringify({ changes: changedFiles });
const overlayChangesFile = path.join(
getTemporaryDirectory(),
"overlay-changes.json",
);
logger.debug(
`Writing overlay changed files to ${overlayChangesFile}: ${changedFilesJson}`,
);
await fs.promises.writeFile(overlayChangesFile, changedFilesJson);
return overlayChangesFile;
}
function computeChangedFiles(
baseFileOids: { [key: string]: string },
overlayFileOids: { [key: string]: string },
): string[] {
const changes: string[] = [];
for (const [file, oid] of Object.entries(overlayFileOids)) {
if (!(file in baseFileOids) || baseFileOids[file] !== oid) {
changes.push(file);
}
}
for (const file of Object.keys(baseFileOids)) {
if (!(file in overlayFileOids)) {
changes.push(file);
}
}
return changes;
}