Refactor setting up CodeQL to handle bundle URLs without tags

This commit is contained in:
Henry Mercer 2023-02-03 19:12:06 +00:00
parent e0fc1c91b2
commit f140af5e28
3 changed files with 320 additions and 197 deletions

224
lib/setup-codeql.js generated
View file

@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod }; return (mod && mod.__esModule) ? mod : { "default": mod };
}; };
Object.defineProperty(exports, "__esModule", { value: true }); Object.defineProperty(exports, "__esModule", { value: true });
exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.getCodeQLSource = exports.convertToSemVer = exports.getBundleVersionFromUrl = exports.tryFindCliVersionDotcomOnly = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0; exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.tryGetFallbackToolcacheVersion = exports.getCodeQLSource = exports.convertToSemVer = exports.tryGetBundleVersionFromUrl = exports.tryFindCliVersionDotcomOnly = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0;
const fs = __importStar(require("fs")); const fs = __importStar(require("fs"));
const path = __importStar(require("path")); const path = __importStar(require("path"));
const perf_hooks_1 = require("perf_hooks"); const perf_hooks_1 = require("perf_hooks");
@ -211,21 +211,30 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger)
} }
return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
} }
function getBundleVersionFromTagName(tagName) { function tryGetBundleVersionFromTagName(tagName, logger) {
const match = tagName.match(/^codeql-bundle-(.*)$/); const match = tagName.match(/^codeql-bundle-(.*)$/);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error(`Malformed bundle tag name: ${tagName}. Bundle version could not be inferred`); logger.debug(`Could not determine bundle version from tag ${tagName}.`);
return undefined;
} }
return match[1]; return match[1];
} }
function getBundleVersionFromUrl(url) { function tryGetTagNameFromUrl(url, logger) {
const match = url.match(/\/(codeql-bundle-.*)\//); const match = url.match(/\/(codeql-bundle-.*)\//);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error(`Malformed tools url: ${url}. Bundle version could not be inferred`); logger.debug(`Could not determine tag name for URL ${url}.`);
return undefined;
} }
return getBundleVersionFromTagName(match[1]); return match[1];
} }
exports.getBundleVersionFromUrl = getBundleVersionFromUrl; function tryGetBundleVersionFromUrl(url, logger) {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
}
exports.tryGetBundleVersionFromUrl = tryGetBundleVersionFromUrl;
function convertToSemVer(version, logger) { function convertToSemVer(version, logger) {
if (!semver.valid(version)) { if (!semver.valid(version)) {
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`); logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
@ -238,18 +247,10 @@ function convertToSemVer(version, logger) {
return s; return s;
} }
exports.convertToSemVer = convertToSemVer; exports.convertToSemVer = convertToSemVer;
async function getOrFindBundleTagName(version, logger) {
if (version.variant === util.GitHubVariant.DOTCOM) {
return await findCodeQLBundleTagDotcomOnly(version.cliVersion, logger);
}
else {
return version.tagName;
}
}
/** /**
* Look for a version of the CodeQL tools in the cache which could override the requested CLI version. * Look for a version of the CodeQL tools in the cache which could override the requested CLI version.
*/ */
async function findOverridingToolsInCache(requestedCliVersion, logger) { async function findOverridingToolsInCache(humanReadableVersion, logger) {
const candidates = toolcache const candidates = toolcache
.findAllVersions("CodeQL") .findAllVersions("CodeQL")
.filter(util_1.isGoodVersion) .filter(util_1.isGoodVersion)
@ -260,7 +261,7 @@ async function findOverridingToolsInCache(requestedCliVersion, logger) {
.filter(({ folder }) => fs.existsSync(path.join(folder, "pinned-version"))); .filter(({ folder }) => fs.existsSync(path.join(folder, "pinned-version")));
if (candidates.length === 1) { if (candidates.length === 1) {
const candidate = candidates[0]; const candidate = candidates[0];
logger.debug(`CodeQL tools version ${candidate.version} in toolcache overriding version ${requestedCliVersion}.`); logger.debug(`CodeQL tools version ${candidate.version} in toolcache overriding version ${humanReadableVersion}.`);
return { return {
codeqlFolder: candidate.folder, codeqlFolder: candidate.folder,
sourceType: "toolcache", sourceType: "toolcache",
@ -300,108 +301,148 @@ async function getCodeQLSource(toolsInput, bypassToolcache, defaultCliVersion, a
if (forceLatest) { if (forceLatest) {
logger.debug(`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`); logger.debug(`Forcing the latest version of the CodeQL tools since ${forceLatestReason}.`);
} }
/** CLI version number, for example 2.12.1. */
let cliVersion;
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
let tagName;
/** /**
* The requested version is: * URL of the CodeQL bundle.
* *
* 1. The one in `defaults.json`, if forceLatest is true. * This does not always include a tag name.
* 2. The version specified by the tools input URL, if one was provided.
* 3. The default CLI version, otherwise.
* We include a `variant` property to let us verify using the type system that
* `tagName` is only undefined when the variant is Dotcom. This lets us ensure
* that we can always compute `tagName`, either by using the existing tag name
* on enterprise instances, or calling `findCodeQLBundleTagDotcomOnly` on
* Dotcom.
*/ */
const requestedVersion = forceLatest let url;
? // case 1 if (forceLatest) {
{ // When forceLatest is true, the requested version is the one shipped with the Action in `defaults.json`.
cliVersion: defaults.cliVersion, cliVersion = defaults.cliVersion;
syntheticCliVersion: defaults.cliVersion, tagName = defaults.bundleVersion;
tagName: defaults.bundleVersion, }
variant, else if (toolsInput !== undefined) {
} // If a tools URL was provided, then use that.
: toolsInput !== undefined tagName = tryGetTagNameFromUrl(toolsInput, logger);
? // case 2 url = toolsInput;
{ }
syntheticCliVersion: convertToSemVer(getBundleVersionFromUrl(toolsInput), logger), else {
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`, // Otherwise, use the default CLI version passed in.
url: toolsInput, cliVersion = defaultCliVersion.cliVersion;
variant, tagName = defaultCliVersion["tagName"];
} }
: // case 3 const bundleVersion = tagName && tryGetBundleVersionFromTagName(tagName, logger);
{ const humanReadableVersion = cliVersion ??
...defaultCliVersion, (bundleVersion && convertToSemVer(bundleVersion, logger)) ??
syntheticCliVersion: defaultCliVersion.cliVersion, tagName ??
}; url ??
// If we find the specified version, we always use that. "unknown";
let codeqlFolder = toolcache.find("CodeQL", requestedVersion.syntheticCliVersion); logger.debug("Attempting to obtain CodeQL tools. " +
let tagName = requestedVersion["tagName"]; `CLI version: ${cliVersion ?? "unknown"}, ` +
if (!codeqlFolder) { `bundle tag name: ${tagName ?? "unknown"}, ` +
logger.debug("Didn't find a version of the CodeQL tools in the toolcache with a version number " + `URL: ${url ?? "unspecified"}.`);
`exactly matching ${requestedVersion.syntheticCliVersion}.`); let codeqlFolder;
if (requestedVersion.cliVersion) { if (cliVersion) {
// If we find the specified CLI version, we always use that.
codeqlFolder = toolcache.find("CodeQL", cliVersion);
// Fall back to matching `x.y.z-<tagName>`.
if (!codeqlFolder) {
logger.debug("Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${cliVersion}.`);
const allVersions = toolcache.findAllVersions("CodeQL"); const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`); logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`);
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is // If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it. // the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) => version.startsWith(`${requestedVersion.cliVersion}-`)); const candidateVersions = allVersions.filter((version) => version.startsWith(`${cliVersion}-`));
if (candidateVersions.length === 1) { if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that."); logger.debug(`Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` +
"toolcache, using that.");
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]); codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} }
else { else {
logger.debug("Did not find exactly one version of the CodeQL tools starting with the requested version."); logger.debug(`Did not find exactly one version of the CodeQL tools starting with ${cliVersion} ` +
`(instead found ${candidateVersions.length}). Trying next fallback method.`);
} }
} }
} }
if (!codeqlFolder && requestedVersion.cliVersion) { // Fall back to matching `0.0.0-<bundleVersion>`.
// Fall back to accepting a `0.0.0-<bundleVersion>` version if we didn't find the if (!codeqlFolder && (cliVersion || tagName)) {
// `x.y.z` version. This is to support old versions of the toolcache. if (cliVersion || tagName) {
// const fallbackVersion = await tryGetFallbackToolcacheVersion(cliVersion, tagName, variant, logger);
// If we are on Dotcom, we will make an HTTP request to the Releases API here if (fallbackVersion) {
// to find the tag name for the requested version. codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
tagName = }
tagName || (await getOrFindBundleTagName(requestedVersion, logger)); else {
const fallbackVersion = convertToSemVer(getBundleVersionFromTagName(tagName), logger); logger.debug("Could not determine a fallback toolcache version number for CodeQL tools version " +
logger.debug(`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL tools version ` + `${humanReadableVersion}.`);
`${requestedVersion.cliVersion}.`); }
codeqlFolder = toolcache.find("CodeQL", fallbackVersion); }
else {
logger.debug("Both the CLI version and the bundle version are unknown, so we will not be able to find " +
"the requested version of the CodeQL tools in the toolcache.");
}
}
if (codeqlFolder) {
logger.info(`Found CodeQL tools version ${humanReadableVersion} in the toolcache.`);
}
else {
logger.info(`Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`);
} }
if (codeqlFolder) { if (codeqlFolder) {
return { return {
codeqlFolder, codeqlFolder,
sourceType: "toolcache", sourceType: "toolcache",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
logger.debug(`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`);
// If we don't find the requested version on Enterprise, we may allow a // If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been // different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it). // specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) { if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) {
const result = await findOverridingToolsInCache(requestedVersion.syntheticCliVersion, logger); const result = await findOverridingToolsInCache(humanReadableVersion, logger);
if (result !== undefined) { if (result !== undefined) {
return result; return result;
} }
} }
if (!url) {
if (!tagName && cliVersion && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
else if (!tagName) {
throw new Error(`Could not obtain the requested version (${humanReadableVersion}) of the CodeQL tools ` +
"since we could not compute the tag name.");
}
url = await getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger);
}
return { return {
cliVersion: requestedVersion.cliVersion || undefined, bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
codeqlURL: requestedVersion["url"] || cliVersion,
(await getCodeQLBundleDownloadURL(tagName || codeqlURL: url,
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)), apiDetails, variant, logger)),
sourceType: "download", sourceType: "download",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
exports.getCodeQLSource = getCodeQLSource; exports.getCodeQLSource = getCodeQLSource;
async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, tempDir, logger) { /**
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
async function tryGetFallbackToolcacheVersion(cliVersion, tagName, variant, logger) {
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
if (cliVersion && !tagName && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
if (!tagName) {
return undefined;
}
const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger);
if (!bundleVersion) {
return undefined;
}
const fallbackVersion = convertToSemVer(bundleVersion, logger);
logger.debug(`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL version ` +
`${cliVersion ?? tagName}.`);
return fallbackVersion;
}
exports.tryGetFallbackToolcacheVersion = tryGetFallbackToolcacheVersion;
async function downloadCodeQL(codeqlURL, maybeBundleVersion, maybeCliVersion, apiDetails, variant, tempDir, logger) {
const parsedCodeQLURL = new URL(codeqlURL); const parsedCodeQLURL = new URL(codeqlURL);
const searchParams = new URLSearchParams(parsedCodeQLURL.search); const searchParams = new URLSearchParams(parsedCodeQLURL.search);
const headers = { const headers = {
@ -430,7 +471,16 @@ async function downloadCodeQL(codeqlURL, maybeCliVersion, apiDetails, variant, t
const toolsDownloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart); const toolsDownloadDurationMs = Math.round(perf_hooks_1.performance.now() - toolsDownloadStart);
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`); logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath); const codeqlExtracted = await toolcache.extractTar(codeqlPath);
const bundleVersion = getBundleVersionFromUrl(codeqlURL); const bundleVersion = maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug("Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`);
return {
toolsVersion: maybeCliVersion ?? "unknown",
codeqlFolder: codeqlExtracted,
toolsDownloadDurationMs,
};
}
// Try to compute the CLI version for this bundle // Try to compute the CLI version for this bundle
const cliVersion = maybeCliVersion || const cliVersion = maybeCliVersion ||
(variant === util.GitHubVariant.DOTCOM && (variant === util.GitHubVariant.DOTCOM &&
@ -494,7 +544,7 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, bypas
toolsSource = init_1.ToolsSource.Toolcache; toolsSource = init_1.ToolsSource.Toolcache;
break; break;
case "download": { case "download": {
const result = await downloadCodeQL(source.codeqlURL, source.cliVersion, apiDetails, variant, tempDir, logger); const result = await downloadCodeQL(source.codeqlURL, source.bundleVersion, source.cliVersion, apiDetails, variant, tempDir, logger);
toolsVersion = result.toolsVersion; toolsVersion = result.toolsVersion;
codeqlFolder = result.codeqlFolder; codeqlFolder = result.codeqlFolder;
toolsDownloadDurationMs = result.toolsDownloadDurationMs; toolsDownloadDurationMs = result.toolsDownloadDurationMs;

File diff suppressed because one or more lines are too long

View file

@ -241,24 +241,36 @@ async function getCodeQLBundleDownloadURL(
return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`; return `https://github.com/${CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
} }
function getBundleVersionFromTagName(tagName: string): string { function tryGetBundleVersionFromTagName(
tagName: string,
logger: Logger
): string | undefined {
const match = tagName.match(/^codeql-bundle-(.*)$/); const match = tagName.match(/^codeql-bundle-(.*)$/);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error( logger.debug(`Could not determine bundle version from tag ${tagName}.`);
`Malformed bundle tag name: ${tagName}. Bundle version could not be inferred` return undefined;
);
} }
return match[1]; return match[1];
} }
export function getBundleVersionFromUrl(url: string): string { function tryGetTagNameFromUrl(url: string, logger: Logger): string | undefined {
const match = url.match(/\/(codeql-bundle-.*)\//); const match = url.match(/\/(codeql-bundle-.*)\//);
if (match === null || match.length < 2) { if (match === null || match.length < 2) {
throw new Error( logger.debug(`Could not determine tag name for URL ${url}.`);
`Malformed tools url: ${url}. Bundle version could not be inferred` return undefined;
);
} }
return getBundleVersionFromTagName(match[1]); return match[1];
}
export function tryGetBundleVersionFromUrl(
url: string,
logger: Logger
): string | undefined {
const tagName = tryGetTagNameFromUrl(url, logger);
if (tagName === undefined) {
return undefined;
}
return tryGetBundleVersionFromTagName(tagName, logger);
} }
export function convertToSemVer(version: string, logger: Logger): string { export function convertToSemVer(version: string, logger: Logger): string {
@ -291,6 +303,8 @@ type CodeQLToolsSource =
toolsVersion: string; toolsVersion: string;
} }
| { | {
/** Bundle version of the tools, if known. */
bundleVersion?: string;
/** CLI version of the tools, if known. */ /** CLI version of the tools, if known. */
cliVersion?: string; cliVersion?: string;
codeqlURL: string; codeqlURL: string;
@ -299,22 +313,11 @@ type CodeQLToolsSource =
toolsVersion: string; toolsVersion: string;
}; };
async function getOrFindBundleTagName(
version: CodeQLDefaultVersionInfo,
logger: Logger
): Promise<string> {
if (version.variant === util.GitHubVariant.DOTCOM) {
return await findCodeQLBundleTagDotcomOnly(version.cliVersion, logger);
} else {
return version.tagName;
}
}
/** /**
* Look for a version of the CodeQL tools in the cache which could override the requested CLI version. * Look for a version of the CodeQL tools in the cache which could override the requested CLI version.
*/ */
async function findOverridingToolsInCache( async function findOverridingToolsInCache(
requestedCliVersion: string, humanReadableVersion: string,
logger: Logger logger: Logger
): Promise<CodeQLToolsSource | undefined> { ): Promise<CodeQLToolsSource | undefined> {
const candidates = toolcache const candidates = toolcache
@ -329,7 +332,7 @@ async function findOverridingToolsInCache(
if (candidates.length === 1) { if (candidates.length === 1) {
const candidate = candidates[0]; const candidate = candidates[0];
logger.debug( logger.debug(
`CodeQL tools version ${candidate.version} in toolcache overriding version ${requestedCliVersion}.` `CodeQL tools version ${candidate.version} in toolcache overriding version ${humanReadableVersion}.`
); );
return { return {
codeqlFolder: candidate.folder, codeqlFolder: candidate.folder,
@ -384,57 +387,59 @@ export async function getCodeQLSource(
); );
} }
/** CLI version number, for example 2.12.1. */
let cliVersion: string | undefined;
/** Tag name of the CodeQL bundle, for example `codeql-bundle-20230120`. */
let tagName: string | undefined;
/** /**
* The requested version is: * URL of the CodeQL bundle.
* *
* 1. The one in `defaults.json`, if forceLatest is true. * This does not always include a tag name.
* 2. The version specified by the tools input URL, if one was provided.
* 3. The default CLI version, otherwise.
* We include a `variant` property to let us verify using the type system that
* `tagName` is only undefined when the variant is Dotcom. This lets us ensure
* that we can always compute `tagName`, either by using the existing tag name
* on enterprise instances, or calling `findCodeQLBundleTagDotcomOnly` on
* Dotcom.
*/ */
const requestedVersion = forceLatest let url: string | undefined;
? // case 1
{
cliVersion: defaults.cliVersion,
syntheticCliVersion: defaults.cliVersion,
tagName: defaults.bundleVersion,
variant,
}
: toolsInput !== undefined
? // case 2
{
syntheticCliVersion: convertToSemVer(
getBundleVersionFromUrl(toolsInput),
logger
),
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`,
url: toolsInput,
variant,
}
: // case 3
{
...defaultCliVersion,
syntheticCliVersion: defaultCliVersion.cliVersion,
};
// If we find the specified version, we always use that. if (forceLatest) {
let codeqlFolder = toolcache.find( // When forceLatest is true, the requested version is the one shipped with the Action in `defaults.json`.
"CodeQL", cliVersion = defaults.cliVersion;
requestedVersion.syntheticCliVersion tagName = defaults.bundleVersion;
} else if (toolsInput !== undefined) {
// If a tools URL was provided, then use that.
tagName = tryGetTagNameFromUrl(toolsInput, logger);
url = toolsInput;
} else {
// Otherwise, use the default CLI version passed in.
cliVersion = defaultCliVersion.cliVersion;
tagName = defaultCliVersion["tagName"];
}
const bundleVersion =
tagName && tryGetBundleVersionFromTagName(tagName, logger);
const humanReadableVersion =
cliVersion ??
(bundleVersion && convertToSemVer(bundleVersion, logger)) ??
tagName ??
url ??
"unknown";
logger.debug(
"Attempting to obtain CodeQL tools. " +
`CLI version: ${cliVersion ?? "unknown"}, ` +
`bundle tag name: ${tagName ?? "unknown"}, ` +
`URL: ${url ?? "unspecified"}.`
); );
let tagName: string | undefined = requestedVersion["tagName"];
if (!codeqlFolder) { let codeqlFolder;
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " + if (cliVersion) {
`exactly matching ${requestedVersion.syntheticCliVersion}.` // If we find the specified CLI version, we always use that.
); codeqlFolder = toolcache.find("CodeQL", cliVersion);
if (requestedVersion.cliVersion) {
// Fall back to matching `x.y.z-<tagName>`.
if (!codeqlFolder) {
logger.debug(
"Didn't find a version of the CodeQL tools in the toolcache with a version number " +
`exactly matching ${cliVersion}.`
);
const allVersions = toolcache.findAllVersions("CodeQL"); const allVersions = toolcache.findAllVersions("CodeQL");
logger.debug( logger.debug(
`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify( `Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(
@ -444,55 +449,72 @@ export async function getCodeQLSource(
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is // If there is exactly one version of the CodeQL tools in the toolcache, and that version is
// the form `x.y.z-<tagName>`, then use it. // the form `x.y.z-<tagName>`, then use it.
const candidateVersions = allVersions.filter((version) => const candidateVersions = allVersions.filter((version) =>
version.startsWith(`${requestedVersion.cliVersion}-`) version.startsWith(`${cliVersion}-`)
); );
if (candidateVersions.length === 1) { if (candidateVersions.length === 1) {
logger.debug("Exactly one candidate version found, using that."); logger.debug(
`Exactly one version of the CodeQL tools starting with ${cliVersion} found in the ` +
"toolcache, using that."
);
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]); codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
} else { } else {
logger.debug( logger.debug(
"Did not find exactly one version of the CodeQL tools starting with the requested version." `Did not find exactly one version of the CodeQL tools starting with ${cliVersion} ` +
`(instead found ${candidateVersions.length}). Trying next fallback method.`
); );
} }
} }
} }
if (!codeqlFolder && requestedVersion.cliVersion) { // Fall back to matching `0.0.0-<bundleVersion>`.
// Fall back to accepting a `0.0.0-<bundleVersion>` version if we didn't find the if (!codeqlFolder && (cliVersion || tagName)) {
// `x.y.z` version. This is to support old versions of the toolcache. if (cliVersion || tagName) {
// const fallbackVersion = await tryGetFallbackToolcacheVersion(
// If we are on Dotcom, we will make an HTTP request to the Releases API here cliVersion,
// to find the tag name for the requested version. tagName,
tagName = variant,
tagName || (await getOrFindBundleTagName(requestedVersion, logger)); logger
const fallbackVersion = convertToSemVer( );
getBundleVersionFromTagName(tagName), if (fallbackVersion) {
logger codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} else {
logger.debug(
"Could not determine a fallback toolcache version number for CodeQL tools version " +
`${humanReadableVersion}.`
);
}
} else {
logger.debug(
"Both the CLI version and the bundle version are unknown, so we will not be able to find " +
"the requested version of the CodeQL tools in the toolcache."
);
}
}
if (codeqlFolder) {
logger.info(
`Found CodeQL tools version ${humanReadableVersion} in the toolcache.`
); );
logger.debug( } else {
`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL tools version ` + logger.info(
`${requestedVersion.cliVersion}.` `Did not find CodeQL tools version ${humanReadableVersion} in the toolcache.`
); );
codeqlFolder = toolcache.find("CodeQL", fallbackVersion);
} }
if (codeqlFolder) { if (codeqlFolder) {
return { return {
codeqlFolder, codeqlFolder,
sourceType: "toolcache", sourceType: "toolcache",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
logger.debug(
`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`
);
// If we don't find the requested version on Enterprise, we may allow a // If we don't find the requested version on Enterprise, we may allow a
// different version to save download time if the version hasn't been // different version to save download time if the version hasn't been
// specified explicitly (in which case we always honor it). // specified explicitly (in which case we always honor it).
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) { if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) {
const result = await findOverridingToolsInCache( const result = await findOverridingToolsInCache(
requestedVersion.syntheticCliVersion, humanReadableVersion,
logger logger
); );
if (result !== undefined) { if (result !== undefined) {
@ -500,30 +522,66 @@ export async function getCodeQLSource(
} }
} }
if (!url) {
if (!tagName && cliVersion && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
} else if (!tagName) {
throw new Error(
`Could not obtain the requested version (${humanReadableVersion}) of the CodeQL tools ` +
"since we could not compute the tag name."
);
}
url = await getCodeQLBundleDownloadURL(
tagName,
apiDetails,
variant,
logger
);
}
return { return {
cliVersion: requestedVersion.cliVersion || undefined, bundleVersion: tagName && tryGetBundleVersionFromTagName(tagName, logger),
codeqlURL: cliVersion,
requestedVersion["url"] || codeqlURL: url,
(await getCodeQLBundleDownloadURL(
tagName ||
// The check on `requestedVersion.tagName` is redundant but lets us
// use the property that if we don't know `requestedVersion.tagName`,
// then we must know `requestedVersion.cliVersion`. This property is
// required by the type of `getOrFindBundleTagName`.
(requestedVersion.tagName !== undefined
? requestedVersion.tagName
: await getOrFindBundleTagName(requestedVersion, logger)),
apiDetails,
variant,
logger
)),
sourceType: "download", sourceType: "download",
toolsVersion: requestedVersion.syntheticCliVersion, toolsVersion: cliVersion ?? humanReadableVersion,
}; };
} }
/**
* Gets a fallback version number to use when looking for CodeQL in the toolcache if we didn't find
* the `x.y.z` version. This is to support old versions of the toolcache.
*/
export async function tryGetFallbackToolcacheVersion(
cliVersion: string | undefined,
tagName: string | undefined,
variant: util.GitHubVariant,
logger: Logger
): Promise<string | undefined> {
//
// If we are on Dotcom, we will make an HTTP request to the Releases API here
// to find the tag name for the requested version.
if (cliVersion && !tagName && variant === util.GitHubVariant.DOTCOM) {
tagName = await findCodeQLBundleTagDotcomOnly(cliVersion, logger);
}
if (!tagName) {
return undefined;
}
const bundleVersion = tryGetBundleVersionFromTagName(tagName, logger);
if (!bundleVersion) {
return undefined;
}
const fallbackVersion = convertToSemVer(bundleVersion, logger);
logger.debug(
`Computed a fallback toolcache version number of ${fallbackVersion} for CodeQL version ` +
`${cliVersion ?? tagName}.`
);
return fallbackVersion;
}
export async function downloadCodeQL( export async function downloadCodeQL(
codeqlURL: string, codeqlURL: string,
maybeBundleVersion: string | undefined,
maybeCliVersion: string | undefined, maybeCliVersion: string | undefined,
apiDetails: api.GitHubApiDetails, apiDetails: api.GitHubApiDetails,
variant: util.GitHubVariant, variant: util.GitHubVariant,
@ -577,7 +635,21 @@ export async function downloadCodeQL(
const codeqlExtracted = await toolcache.extractTar(codeqlPath); const codeqlExtracted = await toolcache.extractTar(codeqlPath);
const bundleVersion = getBundleVersionFromUrl(codeqlURL); const bundleVersion =
maybeBundleVersion ?? tryGetBundleVersionFromUrl(codeqlURL, logger);
if (bundleVersion === undefined) {
logger.debug(
"Could not cache CodeQL tools because we could not determine the bundle version from the " +
`URL ${codeqlURL}.`
);
return {
toolsVersion: maybeCliVersion ?? "unknown",
codeqlFolder: codeqlExtracted,
toolsDownloadDurationMs,
};
}
// Try to compute the CLI version for this bundle // Try to compute the CLI version for this bundle
const cliVersion: string | undefined = const cliVersion: string | undefined =
maybeCliVersion || maybeCliVersion ||
@ -675,6 +747,7 @@ export async function setupCodeQLBundle(
case "download": { case "download": {
const result = await downloadCodeQL( const result = await downloadCodeQL(
source.codeqlURL, source.codeqlURL,
source.bundleVersion,
source.cliVersion, source.cliVersion,
apiDetails, apiDetails,
variant, variant,