robojerk/codeql-action

Merge branch 'main' into add_env_to_config

Browse source
This commit is contained in:
Robert Brignull 2020-08-21 10:32:58 +01:00
commit 0e8b30af75
34 changed files with 378 additions and 143 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/autobuild.ts
View file

@ -1,7 +1,8 @@
import * as core from '@actions/core';
import { getCodeQL, isTracedLanguage } from './codeql';
import { getCodeQL } from './codeql';
import * as config_utils from './config-utils';
import { isTracedLanguage } from './languages';
import * as util from './util';
interface AutobuildStatusReport extends util.StatusReportBase {

21
src/codeql.ts
View file

@ -12,6 +12,7 @@ import uuidV4 from 'uuid/v4';
import * as api from './api-client';
import * as defaults from './defaults.json'; // Referenced from codeql-action-sync-tool!
import { Language } from './languages';
import * as util from './util';
type Options = (string|number|boolean)[];
@ -52,16 +53,16 @@ export interface CodeQL {
/**
* Run 'codeql database init'.
*/
databaseInit(databasePath: string, language: string, sourceRoot: string): Promise<void>;
databaseInit(databasePath: string, language: Language, sourceRoot: string): Promise<void>;
/**
* Runs the autobuilder for the given language.
*/
runAutobuild(language: string): Promise<void>;
runAutobuild(language: Language): Promise<void>;
/**
* Extract code for a scanned language using 'codeql database trace-command'
* and running the language extracter.
*/
extractScannedLanguage(database: string, language: string): Promise<void>;
extractScannedLanguage(database: string, language: Language): Promise<void>;
/**
* Finalize a database using 'codeql database finalize'.
*/
@ -330,7 +331,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
]);
return JSON.parse(fs.readFileSync(envFile, 'utf-8'));
},
databaseInit: async function(databasePath: string, language: string, sourceRoot: string) {
databaseInit: async function(databasePath: string, language: Language, sourceRoot: string) {
await exec.exec(cmd, [
'database',
'init',
@ -340,7 +341,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
...getExtraOptionsFromEnv(['database', 'init']),
]);
},
runAutobuild: async function(language: string) {
runAutobuild: async function(language: Language) {
const cmdName = process.platform === 'win32' ? 'autobuild.cmd' : 'autobuild.sh';
const autobuildCmd = path.join(path.dirname(cmd), language, 'tools', cmdName);
@ -354,7 +355,7 @@ function getCodeQLForCmd(cmd: string): CodeQL {
await exec.exec(autobuildCmd);
},
extractScannedLanguage: async function(databasePath: string, language: string) {
extractScannedLanguage: async function(databasePath: string, language: Language) {
// Get extractor location
let extractorPath = '';
await exec.exec(
@ -435,14 +436,6 @@ function getCodeQLForCmd(cmd: string): CodeQL {
};
}
export function isTracedLanguage(language: string): boolean {
return ['cpp', 'java', 'csharp'].includes(language);
}
export function isScannedLanguage(language: string): boolean {
return !isTracedLanguage(language);
}
/**
* Gets the options for `path` of `options` as an array of extra option strings.
*/

3
src/config-utils.test.ts
View file

@ -7,6 +7,7 @@ import sinon from 'sinon';
import * as api from './api-client';
import { getCachedCodeQL, setCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { Language } from "./languages";
import {setupTests} from './testing-utils';
import * as util from './util';
@ -196,7 +197,7 @@ test("load non-empty input", async t => {
// And the config we expect it to parse to
const expectedConfig: configUtils.Config = {
languages: ['javascript'],
languages: [Language.javascript],
queries: {'javascript': ['/foo/a.ql', '/bar/b.ql']},
pathsIgnore: ['a', 'b'],
paths: ['c/d'],

48
src/config-utils.ts
View file

@ -6,6 +6,7 @@ import * as path from 'path';
import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import * as util from './util';
// Property names from the user-supplied config file.
@ -16,16 +17,6 @@ const QUERIES_USES_PROPERTY = 'uses';
const PATHS_IGNORE_PROPERTY = 'paths-ignore';
const PATHS_PROPERTY = 'paths';
// All the languages supported by CodeQL
const ALL_LANGUAGES = ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] as const;
type Language = (typeof ALL_LANGUAGES)[number];
// Some alternate names for languages
const LANGUAGE_ALIASES: {[name: string]: Language} = {
'c': 'cpp',
'typescript': 'javascript',
};
/**
* Format of the config file supplied by the user.
*/
@ -448,17 +439,6 @@ export function getUnknownLanguagesError(languages: string[]): string {
* Gets the set of languages in the current repository
*/
async function getLanguagesInRepo(): Promise<Language[]> {
// Translate between GitHub's API names for languages and ours
const codeqlLanguages: {[lang: string]: Language} = {
'C': 'cpp',
'C++': 'cpp',
'C#': 'csharp',
'Go': 'go',
'Java': 'java',
'JavaScript': 'javascript',
'TypeScript': 'javascript',
'Python': 'python',
};
let repo_nwo = process.env['GITHUB_REPOSITORY']?.split("/");
if (repo_nwo) {
let owner = repo_nwo[0];
@ -477,9 +457,10 @@ async function getLanguagesInRepo(): Promise<Language[]> {
// Since sets in javascript maintain insertion order, using a set here and then splatting it
// into an array gives us an array of languages ordered by popularity
let languages: Set<Language> = new Set();
for (let lang in response.data) {
if (lang in codeqlLanguages) {
languages.add(codeqlLanguages[lang]);
for (let lang of Object.keys(response.data)) {
let parsedLang = parseLanguage(lang);
if (parsedLang !== undefined) {
languages.add(parsedLang);
}
}
return [...languages];
@ -520,28 +501,21 @@ async function getLanguages(): Promise<Language[]> {
}
// Make sure they are supported
const checkedLanguages: Language[] = [];
const parsedLanguages: Language[] = [];
const unknownLanguages: string[] = [];
for (let language of languages) {
// Normalise to lower case
language = language.toLowerCase();
// Resolve any known aliases
if (language in LANGUAGE_ALIASES) {
language = LANGUAGE_ALIASES[language];
}
const checkedLanguage = ALL_LANGUAGES.find(l => l === language);
if (checkedLanguage === undefined) {
const parsedLanguage = parseLanguage(language);
if (parsedLanguage === undefined) {
unknownLanguages.push(language);
} else if (checkedLanguages.indexOf(checkedLanguage) === -1) {
checkedLanguages.push(checkedLanguage);
} else if (parsedLanguages.indexOf(parsedLanguage) === -1) {
parsedLanguages.push(parsedLanguage);
}
}
if (unknownLanguages.length > 0) {
throw new Error(getUnknownLanguagesError(unknownLanguages));
}
return checkedLanguages;
return parsedLanguages;
}
/**

3
src/finalize-db.ts
View file

@ -2,8 +2,9 @@ import * as core from '@actions/core';
import * as fs from 'fs';
import * as path from 'path';
import { getCodeQL, isScannedLanguage } from './codeql';
import { getCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isScannedLanguage } from './languages';
import { getActionsLogger } from './logging';
import { parseRepositoryNwo } from './repository';
import * as sharedEnv from './shared-environment';

7
src/fingerprints.test.ts
View file

@ -4,6 +4,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as fingerprints from './fingerprints';
import { getCLILogger } from './logging';
import {setupTests} from './testing-utils';
setupTests(test);
@ -115,7 +116,7 @@ test('hash', (t: ava.Assertions) => {
function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {
const location = { "uri": uri, "index": index };
const artifacts = artifactsURIs.map(uri => ({ "location": { "uri": uri } }));
return fingerprints.resolveUriToFile(location, artifacts);
return fingerprints.resolveUriToFile(location, artifacts, getCLILogger());
}
test('resolveUriToFile', t => {
@ -174,7 +175,7 @@ test('addFingerprints', t => {
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input), expected);
t.deepEqual(fingerprints.addFingerprints(input, getCLILogger()), expected);
});
test('missingRegions', t => {
@ -189,5 +190,5 @@ test('missingRegions', t => {
// The URIs in the SARIF files resolve to files in the testdata directory
process.env['GITHUB_WORKSPACE'] = path.normalize(__dirname + '/../src/testdata');
t.deepEqual(fingerprints.addFingerprints(input), expected);
t.deepEqual(fingerprints.addFingerprints(input, getCLILogger()), expected);
});

27
src/fingerprints.ts
View file

@ -1,7 +1,8 @@
import * as core from '@actions/core';
import * as fs from 'fs';
import Long from 'long';
import { Logger } from './logging';
const tab = '\t'.charCodeAt(0);
const space = ' '.charCodeAt(0);
const lf = '\n'.charCodeAt(0);
@ -124,7 +125,7 @@ export function hash(callback: hashCallback, input: string) {
// Generate a hash callback function that updates the given result in-place
// when it recieves a hash for the correct line number. Ignores hashes for other lines.
function locationUpdateCallback(result: any, location: any): hashCallback {
function locationUpdateCallback(result: any, location: any, logger: Logger): hashCallback {
let locationStartLine = location.physicalLocation?.region?.startLine;
if (locationStartLine === undefined) {
// We expect the region section to be present, but it can be absent if the
@ -148,7 +149,7 @@ function locationUpdateCallback(result: any, location: any): hashCallback {
if (!existingFingerprint) {
result.partialFingerprints.primaryLocationLineHash = hash;
} else if (existingFingerprint !== hash) {
core.warning('Calculated fingerprint of ' + hash +
logger.warning('Calculated fingerprint of ' + hash +
' for file ' + location.physicalLocation.artifactLocation.uri +
' line ' + lineNumber +
', but found existing inconsistent fingerprint value ' + existingFingerprint);
@ -160,14 +161,14 @@ function locationUpdateCallback(result: any, location: any): hashCallback {
// the source file so we can hash it.
// If possible returns a absolute file path for the source file,
// or if not possible then returns undefined.
export function resolveUriToFile(location: any, artifacts: any[]): string | undefined {
export function resolveUriToFile(location: any, artifacts: any[], logger: Logger): string | undefined {
// This may be referencing an artifact
if (!location.uri && location.index !== undefined) {
if (typeof location.index !== 'number' ||
location.index < 0 ||
location.index >= artifacts.length ||
typeof artifacts[location.index].location !== 'object') {
core.debug(`Ignoring location as URI "${location.index}" is invalid`);
logger.debug(`Ignoring location as URI "${location.index}" is invalid`);
return undefined;
}
location = artifacts[location.index].location;
@ -175,7 +176,7 @@ export function resolveUriToFile(location: any, artifacts: any[]): string | unde
// Get the URI and decode
if (typeof location.uri !== 'string') {
core.debug(`Ignoring location as index "${location.uri}" is invalid`);
logger.debug(`Ignoring location as index "${location.uri}" is invalid`);
return undefined;
}
let uri = decodeURIComponent(location.uri);
@ -186,14 +187,14 @@ export function resolveUriToFile(location: any, artifacts: any[]): string | unde
uri = uri.substring(fileUriPrefix.length);
}
if (uri.indexOf('://') !== -1) {
core.debug(`Ignoring location URI "${uri}" as the scheme is not recognised`);
logger.debug(`Ignoring location URI "${uri}" as the scheme is not recognised`);
return undefined;
}
// Discard any absolute paths that aren't in the src root
const srcRootPrefix = process.env['GITHUB_WORKSPACE'] + '/';
if (uri.startsWith('/') && !uri.startsWith(srcRootPrefix)) {
core.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
logger.debug(`Ignoring location URI "${uri}" as it is outside of the src root`);
return undefined;
}
@ -206,7 +207,7 @@ export function resolveUriToFile(location: any, artifacts: any[]): string | unde
// Check the file exists
if (!fs.existsSync(uri)) {
core.debug(`Unable to compute fingerprint for non-existent file: ${uri}`);
logger.debug(`Unable to compute fingerprint for non-existent file: ${uri}`);
return undefined;
}
@ -215,7 +216,7 @@ export function resolveUriToFile(location: any, artifacts: any[]): string | unde
// Compute fingerprints for results in the given sarif file
// and return an updated sarif file contents.
export function addFingerprints(sarifContents: string): string {
export function addFingerprints(sarifContents: string, logger: Logger): string {
let sarif = JSON.parse(sarifContents);
// Gather together results for the same file and construct
@ -229,18 +230,18 @@ export function addFingerprints(sarifContents: string): string {
// Check the primary location is defined correctly and is in the src root
const primaryLocation = (result.locations || [])[0];
if (!primaryLocation?.physicalLocation?.artifactLocation) {
core.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
logger.debug(`Unable to compute fingerprint for invalid location: ${JSON.stringify(primaryLocation)}`);
continue;
}
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts);
const filepath = resolveUriToFile(primaryLocation.physicalLocation.artifactLocation, artifacts, logger);
if (!filepath) {
continue;
}
if (!callbacksByFile[filepath]) {
callbacksByFile[filepath] = [];
}
callbacksByFile[filepath].push(locationUpdateCallback(result, primaryLocation));
callbacksByFile[filepath].push(locationUpdateCallback(result, primaryLocation, logger));
}
}

47
src/languages.test.ts Normal file
View file

@ -0,0 +1,47 @@
import test from 'ava';
import {isScannedLanguage, isTracedLanguage, Language, parseLanguage} from './languages';
import {setupTests} from './testing-utils';
setupTests(test);
test('parseLangauge', async t => {
// Exact matches
t.deepEqual(parseLanguage('csharp'), Language.csharp);
t.deepEqual(parseLanguage('cpp'), Language.cpp);
t.deepEqual(parseLanguage('go'), Language.go);
t.deepEqual(parseLanguage('java'), Language.java);
t.deepEqual(parseLanguage('javascript'), Language.javascript);
t.deepEqual(parseLanguage('python'), Language.python);
// Aliases
t.deepEqual(parseLanguage('c'), Language.cpp);
t.deepEqual(parseLanguage('c++'), Language.cpp);
t.deepEqual(parseLanguage('c#'), Language.csharp);
t.deepEqual(parseLanguage('typescript'), Language.javascript);
// Not matches
t.deepEqual(parseLanguage('foo'), undefined);
t.deepEqual(parseLanguage(' '), undefined);
t.deepEqual(parseLanguage(''), undefined);
});
test('isTracedLanguage', async t => {
t.true(isTracedLanguage(Language.cpp));
t.true(isTracedLanguage(Language.java));
t.true(isTracedLanguage(Language.csharp));
t.false(isTracedLanguage(Language.go));
t.false(isTracedLanguage(Language.javascript));
t.false(isTracedLanguage(Language.python));
});
test('isScannedLanguage', async t => {
t.false(isScannedLanguage(Language.cpp));
t.false(isScannedLanguage(Language.java));
t.false(isScannedLanguage(Language.csharp));
t.true(isScannedLanguage(Language.go));
t.true(isScannedLanguage(Language.javascript));
t.true(isScannedLanguage(Language.python));
});

44
src/languages.ts Normal file
View file

@ -0,0 +1,44 @@
// All the languages supported by CodeQL
export enum Language {
csharp = 'csharp',
cpp = 'cpp',
go = 'go',
java = 'java',
javascript = 'javascript',
python = 'python',
}
// Additional names for languages
const LANGUAGE_ALIASES: {[lang: string]: Language} = {
'c': Language.cpp,
'c++': Language.cpp,
'c#': Language.csharp,
'typescript': Language.javascript,
};
// Translate from user input or GitHub's API names for languages to CodeQL's names for languages
export function parseLanguage(language: string): Language | undefined {
// Normalise to lower case
language = language.toLowerCase();
// See if it's an exact match
if (language in Language) {
return language as Language;
}
// Check language aliases
if (language in LANGUAGE_ALIASES) {
return LANGUAGE_ALIASES[language];
}
return undefined;
}
export function isTracedLanguage(language: Language): boolean {
return ['cpp', 'java', 'csharp'].includes(language);
}
export function isScannedLanguage(language: Language): boolean {
return !isTracedLanguage(language);
}

3
src/setup-tracer.ts
View file

@ -4,8 +4,9 @@ import * as fs from 'fs';
import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, isTracedLanguage, setupCodeQL } from './codeql';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import { isTracedLanguage } from './languages';
import * as util from './util';
type TracerConfig = {

2
src/upload-lib.ts
View file

@ -235,7 +235,7 @@ async function uploadFiles(
}
let sarifPayload = combineSarifFiles(sarifFiles);
sarifPayload = fingerprints.addFingerprints(sarifPayload);
sarifPayload = fingerprints.addFingerprints(sarifPayload, logger);
const zipped_sarif = zlib.gzipSync(sarifPayload).toString('base64');
let checkoutURI = fileUrl(checkoutPath);