Cache explicitly requested bundles with their URL if possible
This commit is contained in:
Henry Mercer
parent
c9b1be5115
commit
c2e39e078f
9 changed files with 396 additions and 154 deletions
154
lib/setup-codeql.js
generated
154
lib/setup-codeql.js
generated
|
|
@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.getCodeQLSource = exports.convertToSemVer = exports.getBundleTagNameFromUrl = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0;
|
||||
exports.setupCodeQLBundle = exports.getCodeQLURLVersion = exports.downloadCodeQL = exports.getCodeQLSource = exports.convertToSemVer = exports.getBundleVersionFromUrl = exports.tryFindCliVersionDotcomOnly = exports.findCodeQLBundleTagDotcomOnly = exports.getCodeQLActionRepository = exports.CODEQL_DEFAULT_ACTION_REPOSITORY = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const toolcache = __importStar(require("@actions/tool-cache"));
|
||||
|
|
@ -66,17 +66,21 @@ function getCodeQLActionRepository(logger) {
|
|||
}
|
||||
exports.getCodeQLActionRepository = getCodeQLActionRepository;
|
||||
/**
|
||||
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
|
||||
* to directly find the CodeQL bundle release for a particular CLI version.
|
||||
* Gets the tag name and, if known, the CodeQL CLI version for each CodeQL bundle release.
|
||||
*
|
||||
* To get around this, we add a `codeql-version-x.y.z.txt` asset to each bundle release that
|
||||
* specifies the CLI version for that bundle release.
|
||||
* CodeQL bundles are currently tagged in the form `codeql-bundle-yyyymmdd`, so it is not possible
|
||||
* to directly find the CodeQL bundle release for a particular CLI version or find the CodeQL CLI
|
||||
* version for a particular CodeQL bundle.
|
||||
*
|
||||
* To get around this, we add a `cli-version-x.y.z.txt` asset to each bundle release that specifies
|
||||
* the CLI version for that bundle release. We can then use the GitHub Releases for the CodeQL
|
||||
* Action as a source of truth.
|
||||
*
|
||||
* In the medium term, we should migrate to a tagging scheme that allows us to directly find the
|
||||
* CodeQL bundle release for a particular CLI version, for example `codeql-bundle-vx.y.z`.
|
||||
*/
|
||||
async function findCodeQLBundleTagDotcomOnly(cliVersion, logger) {
|
||||
logger.debug(`Trying to find the CodeQL bundle release for CLI version ${cliVersion}.`);
|
||||
async function getCodeQLBundleReleasesDotcomOnly(logger) {
|
||||
logger.debug(`Fetching CodeQL CLI version and CodeQL bundle tag name information for releases of the CodeQL tools.`);
|
||||
const apiClient = api.getApiClient();
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
const releases = await apiClient.paginate(apiClient.repos.listReleases, {
|
||||
|
|
@ -84,26 +88,65 @@ async function findCodeQLBundleTagDotcomOnly(cliVersion, logger) {
|
|||
repo: codeQLActionRepository.split("/")[1],
|
||||
});
|
||||
logger.debug(`Found ${releases.length} releases.`);
|
||||
for (const release of releases) {
|
||||
return releases.flatMap((release) => {
|
||||
const cliVersionFileVersions = release.assets
|
||||
.map((asset) => { var _a; return (_a = asset.name.match(/cli-version-(.*)\.txt/)) === null || _a === void 0 ? void 0 : _a[1]; })
|
||||
.filter((v) => v)
|
||||
.map((v) => v);
|
||||
if (cliVersionFileVersions.length === 0) {
|
||||
logger.debug(`Ignoring release ${release.tag_name} with no CLI version marker file.`);
|
||||
continue;
|
||||
}
|
||||
if (cliVersionFileVersions.length > 1) {
|
||||
logger.warning(`Ignoring release ${release.tag_name} with multiple CLI version marker files.`);
|
||||
continue;
|
||||
}
|
||||
if (cliVersionFileVersions[0] === cliVersion) {
|
||||
return release.tag_name;
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{ cliVersion: cliVersionFileVersions[0], tagName: release.tag_name },
|
||||
];
|
||||
});
|
||||
}
|
||||
async function tryGetCodeQLCliVersionForRelease(release, logger) {
|
||||
const cliVersionsFromMarkerFiles = release.assets
|
||||
.map((asset) => { var _a; return (_a = asset.name.match(/cli-version-(.*)\.txt/)) === null || _a === void 0 ? void 0 : _a[1]; })
|
||||
.filter((v) => v)
|
||||
.map((v) => v);
|
||||
if (cliVersionsFromMarkerFiles.length > 1) {
|
||||
logger.warning(`Ignoring release ${release.tag_name} with multiple CLI version marker files.`);
|
||||
return undefined;
|
||||
}
|
||||
throw new Error(`Failed to find a CodeQL bundle release for CLI version ${cliVersion}.`);
|
||||
else if (cliVersionsFromMarkerFiles.length === 0) {
|
||||
logger.debug(`Failed to find the CodeQL CLI version for release ${release.tag_name}.`);
|
||||
return undefined;
|
||||
}
|
||||
return cliVersionsFromMarkerFiles[0];
|
||||
}
|
||||
async function findCodeQLBundleTagDotcomOnly(cliVersion, logger) {
|
||||
const filtered = (await getCodeQLBundleReleasesDotcomOnly(logger)).filter((release) => release.cliVersion === cliVersion);
|
||||
if (filtered.length === 0) {
|
||||
throw new Error(`Failed to find a release of the CodeQL tools that contains CodeQL CLI ${cliVersion}.`);
|
||||
}
|
||||
else if (filtered.length > 1) {
|
||||
throw new Error(`Found multiple releases of the CodeQL tools that contain CodeQL CLI ${cliVersion}. ` +
|
||||
`Only one such release should exist.`);
|
||||
}
|
||||
return filtered[0].tagName;
|
||||
}
|
||||
exports.findCodeQLBundleTagDotcomOnly = findCodeQLBundleTagDotcomOnly;
|
||||
async function tryFindCliVersionDotcomOnly(tagName, logger) {
|
||||
try {
|
||||
logger.debug(`Fetching the GitHub Release for the CodeQL bundle tagged ${tagName}.`);
|
||||
const apiClient = api.getApiClient();
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
const release = await apiClient.repos.getReleaseByTag({
|
||||
owner: codeQLActionRepository.split("/")[0],
|
||||
repo: codeQLActionRepository.split("/")[1],
|
||||
tag: tagName,
|
||||
});
|
||||
return tryGetCodeQLCliVersionForRelease(release.data, logger);
|
||||
}
|
||||
catch (e) {
|
||||
logger.error(`Failed to find the CLI version for the CodeQL bundle tagged ${tagName}. Error: ${e instanceof Error ? e.message : e}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
exports.tryFindCliVersionDotcomOnly = tryFindCliVersionDotcomOnly;
|
||||
async function getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger) {
|
||||
const codeQLActionRepository = getCodeQLActionRepository(logger);
|
||||
const potentialDownloadSources = [
|
||||
|
|
@ -171,14 +214,14 @@ async function getCodeQLBundleDownloadURL(tagName, apiDetails, variant, logger)
|
|||
}
|
||||
return `https://github.com/${exports.CODEQL_DEFAULT_ACTION_REPOSITORY}/releases/download/${tagName}/${codeQLBundleName}`;
|
||||
}
|
||||
function getBundleTagNameFromUrl(url) {
|
||||
function getBundleVersionFromUrl(url) {
|
||||
const match = url.match(/\/codeql-bundle-(.*)\//);
|
||||
if (match === null || match.length < 2) {
|
||||
throw new Error(`Malformed tools url: ${url}. Tag name could not be inferred`);
|
||||
throw new Error(`Malformed tools url: ${url}. Bundle version could not be inferred`);
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
exports.getBundleTagNameFromUrl = getBundleTagNameFromUrl;
|
||||
exports.getBundleVersionFromUrl = getBundleVersionFromUrl;
|
||||
function convertToSemVer(version, logger) {
|
||||
if (!semver.valid(version)) {
|
||||
logger.debug(`Bundle version ${version} is not in SemVer format. Will treat it as pre-release 0.0.0-${version}.`);
|
||||
|
|
@ -270,39 +313,45 @@ async function getCodeQLSource(toolsInput, bypassToolcache, defaultCliVersion, a
|
|||
? // case 1
|
||||
{
|
||||
cliVersion: defaults.cliVersion,
|
||||
syntheticCliVersion: defaults.cliVersion,
|
||||
tagName: defaults.bundleVersion,
|
||||
variant,
|
||||
}
|
||||
: toolsInput !== undefined
|
||||
? // case 2
|
||||
{
|
||||
cliVersion: convertToSemVer(getBundleTagNameFromUrl(toolsInput), logger),
|
||||
tagName: getBundleTagNameFromUrl(toolsInput),
|
||||
syntheticCliVersion: convertToSemVer(getBundleVersionFromUrl(toolsInput), logger),
|
||||
tagName: `codeql-bundle-${getBundleVersionFromUrl(toolsInput)}`,
|
||||
url: toolsInput,
|
||||
variant,
|
||||
}
|
||||
: // case 3
|
||||
defaultCliVersion;
|
||||
{
|
||||
...defaultCliVersion,
|
||||
syntheticCliVersion: defaultCliVersion.cliVersion,
|
||||
};
|
||||
// If we find the specified version, we always use that.
|
||||
let codeqlFolder = toolcache.find("CodeQL", requestedVersion.cliVersion);
|
||||
let codeqlFolder = toolcache.find("CodeQL", requestedVersion.syntheticCliVersion);
|
||||
let tagName = requestedVersion["tagName"];
|
||||
if (!codeqlFolder) {
|
||||
logger.debug("Didn't find a version of the CodeQL tools in the toolcache with a version number " +
|
||||
`exactly matching ${requestedVersion.cliVersion}.`);
|
||||
const allVersions = toolcache.findAllVersions("CodeQL");
|
||||
logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`);
|
||||
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
|
||||
// the form `x.y.z-<tagName>`, then use it.
|
||||
const candidateVersions = allVersions.filter((version) => version.startsWith(`${requestedVersion.cliVersion}-`));
|
||||
if (candidateVersions.length === 1) {
|
||||
logger.debug("Exactly one candidate version found, using that.");
|
||||
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
|
||||
}
|
||||
else {
|
||||
logger.debug("Did not find exactly one version of the CodeQL tools starting with the requested version.");
|
||||
`exactly matching ${requestedVersion.syntheticCliVersion}.`);
|
||||
if (requestedVersion.cliVersion) {
|
||||
const allVersions = toolcache.findAllVersions("CodeQL");
|
||||
logger.debug(`Found the following versions of the CodeQL tools in the toolcache: ${JSON.stringify(allVersions)}.`);
|
||||
// If there is exactly one version of the CodeQL tools in the toolcache, and that version is
|
||||
// the form `x.y.z-<tagName>`, then use it.
|
||||
const candidateVersions = allVersions.filter((version) => version.startsWith(`${requestedVersion.cliVersion}-`));
|
||||
if (candidateVersions.length === 1) {
|
||||
logger.debug("Exactly one candidate version found, using that.");
|
||||
codeqlFolder = toolcache.find("CodeQL", candidateVersions[0]);
|
||||
}
|
||||
else {
|
||||
logger.debug("Did not find exactly one version of the CodeQL tools starting with the requested version.");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!codeqlFolder && !requestedVersion.cliVersion.startsWith("0.0.0")) {
|
||||
if (!codeqlFolder && requestedVersion.cliVersion) {
|
||||
// Fall back to accepting a `0.0.0-<tagName>` version if we didn't find the
|
||||
// `x.y.z` version. This is to support old versions of the toolcache.
|
||||
//
|
||||
|
|
@ -319,29 +368,36 @@ async function getCodeQLSource(toolsInput, bypassToolcache, defaultCliVersion, a
|
|||
return {
|
||||
codeqlFolder,
|
||||
sourceType: "toolcache",
|
||||
toolsVersion: requestedVersion.cliVersion,
|
||||
toolsVersion: requestedVersion.syntheticCliVersion,
|
||||
};
|
||||
}
|
||||
logger.debug(`Did not find CodeQL tools version ${requestedVersion.cliVersion} in the toolcache.`);
|
||||
logger.debug(`Did not find CodeQL tools version ${requestedVersion.syntheticCliVersion} in the toolcache.`);
|
||||
// If we don't find the requested version on Enterprise, we may allow a
|
||||
// different version to save download time if the version hasn't been
|
||||
// specified explicitly (in which case we always honor it).
|
||||
if (variant !== util.GitHubVariant.DOTCOM && !forceLatest && !toolsInput) {
|
||||
const result = await findOverridingToolsInCache(requestedVersion.cliVersion, logger);
|
||||
const result = await findOverridingToolsInCache(requestedVersion.syntheticCliVersion, logger);
|
||||
if (result !== undefined) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return {
|
||||
cliVersion: requestedVersion.cliVersion || undefined,
|
||||
codeqlURL: requestedVersion["url"] ||
|
||||
(await getCodeQLBundleDownloadURL(tagName || (await getOrFindBundleTagName(requestedVersion, logger)), apiDetails, variant, logger)),
|
||||
semanticVersion: requestedVersion.cliVersion,
|
||||
(await getCodeQLBundleDownloadURL(tagName ||
|
||||
// The check on `requestedVersion.tagName` is redundant but lets us
|
||||
// use the property that if we don't know `requestedVersion.tagName`,
|
||||
// then we must know `requestedVersion.cliVersion`. This property is
|
||||
// required by the type of `getOrFindBundleTagName`.
|
||||
(requestedVersion.tagName !== undefined
|
||||
? requestedVersion.tagName
|
||||
: await getOrFindBundleTagName(requestedVersion, logger)), apiDetails, variant, logger)),
|
||||
sourceType: "download",
|
||||
toolsVersion: requestedVersion.cliVersion,
|
||||
toolsVersion: requestedVersion.syntheticCliVersion,
|
||||
};
|
||||
}
|
||||
exports.getCodeQLSource = getCodeQLSource;
|
||||
async function downloadCodeQL(codeqlURL, semanticVersion, apiDetails, tempDir, logger) {
|
||||
async function downloadCodeQL(codeqlURL, cliVersion, apiDetails, variant, tempDir, logger) {
|
||||
const parsedCodeQLURL = new URL(codeqlURL);
|
||||
const searchParams = new URLSearchParams(parsedCodeQLURL.search);
|
||||
const headers = {
|
||||
|
|
@ -367,7 +423,13 @@ async function downloadCodeQL(codeqlURL, semanticVersion, apiDetails, tempDir, l
|
|||
const codeqlPath = await toolcache.downloadTool(codeqlURL, dest, undefined, finalHeaders);
|
||||
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
|
||||
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
|
||||
return await toolcache.cacheDir(codeqlExtracted, "CodeQL", semanticVersion);
|
||||
const bundleVersion = getBundleVersionFromUrl(codeqlURL);
|
||||
// If we have a CLI version, use that. Otherwise, try to find the CLI version from the GitHub Releases
|
||||
const toolcacheVersion = cliVersion ||
|
||||
(variant === util.GitHubVariant.DOTCOM &&
|
||||
(await tryFindCliVersionDotcomOnly(`codeql-bundle-${bundleVersion}`, logger))) ||
|
||||
convertToSemVer(bundleVersion, logger);
|
||||
return await toolcache.cacheDir(codeqlExtracted, "CodeQL", toolcacheVersion);
|
||||
}
|
||||
exports.downloadCodeQL = downloadCodeQL;
|
||||
function getCodeQLURLVersion(url) {
|
||||
|
|
@ -404,7 +466,7 @@ async function setupCodeQLBundle(toolsInput, apiDetails, tempDir, variant, bypas
|
|||
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
|
||||
break;
|
||||
case "download":
|
||||
codeqlFolder = await downloadCodeQL(source.codeqlURL, source.semanticVersion, apiDetails, tempDir, logger);
|
||||
codeqlFolder = await downloadCodeQL(source.codeqlURL, source.cliVersion, apiDetails, variant, tempDir, logger);
|
||||
break;
|
||||
default:
|
||||
util.assertNever(source);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue