Merge branch 'master' into autobuild_errors

.github/pull_request_template.md

@@ -1,7 +1,7 @@
 ### Merge / deployment checklist
 
 - Run test builds as necessary. Can be on this repository or elsewhere as needed in order to test the change - please include links to tests in other repos!
-  - [ ] CodeQL using init/finish actions
+  - [ ] CodeQL using init/analyze actions
   - [ ] 3rd party tool using upload action
 - [ ] Confirm this change is backwards compatible with existing workflows.
 - [ ] Confirm the [readme](https://github.com/github/codeql-action/blob/master/README.md) has been updated if necessary.

README.md

@@ -2,8 +2,6 @@
 
 This action runs GitHub's industry-leading static analysis engine, CodeQL, against a repository's source code to find security vulnerabilities. It then automatically uploads the results to GitHub so they can be displayed in the repository's security tab. CodeQL runs an extensible set of [queries](https://github.com/semmle/ql), which have been developed by the community and the [GitHub Security Lab](https://securitylab.github.com/) to find common vulnerabilities in your code.
 
-[Sign up for the Advanced Security beta](https://github.com/features/security/advanced-security/signup)
-
 ## Usage
 
 To get code scanning results from CodeQL analysis on your repo you can use the following workflow as a template:
@@ -82,6 +80,8 @@ The CodeQL action should be run on `push` events, and on a `schedule`. `Push` ev
 
 You may optionally specify additional queries for CodeQL to execute by using a config file. The queries must belong to a [QL pack](https://help.semmle.com/codeql/codeql-cli/reference/qlpack-overview.html) and can be in your repository or any public repository. You can choose a single .ql file, a folder containing multiple .ql files, a .qls [query suite](https://help.semmle.com/codeql/codeql-cli/procedures/query-suites.html) file, or any combination of the above. To use queries from other repositories use the same syntax as when [using an action](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstepsuses).
 
+You can disable the default queries using `disable-default-queries: true`.
+
 You can choose to ignore some files or folders from the analysis, or include additional files/folders for analysis. This *only* works for Javascript and Python analysis.
 Identifying potential files for extraction:
 
@@ -102,6 +102,8 @@ A config file looks like this:
 ```yaml
 name: "My CodeQL config"
 
+disable-default-queries: true
+
 queries:
   - name: In-repo queries (Runs the queries located in the my-queries folder of the repo)
     uses: ./my-queries

lib/config-utils.js

@@ -23,6 +23,7 @@ exports.ExternalQuery = ExternalQuery;
 class Config {
     constructor() {
         this.name = "";
+        this.disableDefaultQueries = false;
         this.additionalQueries = [];
         this.externalQueries = [];
         this.pathsIgnore = [];
@@ -75,6 +76,9 @@ function initConfig() {
         if (parsedYAML.name && typeof parsedYAML.name === "string") {
             config.name = parsedYAML.name;
         }
+        if (parsedYAML['disable-default-queries'] && typeof parsedYAML['disable-default-queries'] === "boolean") {
+            config.disableDefaultQueries = parsedYAML['disable-default-queries'];
+        }
         const queries = parsedYAML.queries;
         if (queries && queries instanceof Array) {
             queries.forEach(query => {

lib/finalize-db.js

@@ -73,12 +73,12 @@ async function resolveQueryLanguages(codeqlCmd, config) {
         const noDeclaredLanguage = resolveQueriesOutputObject.noDeclaredLanguage;
         const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
         if (noDeclaredLanguageQueries.length !== 0) {
-            core.warning('Some queries do not declare a language:\n' + noDeclaredLanguageQueries.join('\n'));
+            throw new Error('Some queries do not declare a language, their qlpack.yml file is missing or is invalid');
         }
         const multipleDeclaredLanguages = resolveQueriesOutputObject.multipleDeclaredLanguages;
         const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
         if (multipleDeclaredLanguagesQueries.length !== 0) {
-            core.warning('Some queries declare multiple languages:\n' + multipleDeclaredLanguagesQueries.join('\n'));
+            throw new Error('Some queries declare multiple languages, their qlpack.yml file is missing or is invalid');
         }
     }
     return res;
@@ -88,7 +88,11 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
     const queriesPerLanguage = await resolveQueryLanguages(codeqlCmd, config);
     for (let database of fs.readdirSync(databaseFolder)) {
         core.startGroup('Analyzing ' + database);
-        const additionalQueries = queriesPerLanguage[database] || [];
+        const queries = [];
+        if (!config.disableDefaultQueries) {
+            queries.push(database + '-code-scanning.qls');
+        }
+        queries.push(...(queriesPerLanguage[database] || []));
         const sarifFile = path.join(sarifFolder, database + '.sarif');
         await exec.exec(codeqlCmd, [
             'database',
@@ -97,8 +101,7 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
             '--format=sarif-latest',
             '--output=' + sarifFile,
             '--no-sarif-add-snippets',
-            database + '-code-scanning.qls',
-            ...additionalQueries,
+            ...queries
         ]);
         core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');
         core.endGroup();

lib/upload-lib.js

@@ -54,6 +54,52 @@ function combineSarifFiles(sarifFiles) {
     return JSON.stringify(combinedSarif);
 }
 exports.combineSarifFiles = combineSarifFiles;
+// Upload the given payload.
+// If the request fails then this will retry a small number of times.
+async function uploadPayload(payload) {
+    core.info('Uploading results');
+    const githubToken = core.getInput('token');
+    const ph = new auth.BearerCredentialHandler(githubToken);
+    const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
+    const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
+    // Make up to 4 attempts to upload, and sleep for these
+    // number of seconds between each attempt.
+    // We don't want to backoff too much to avoid wasting action
+    // minutes, but just waiting a little bit could maybe help.
+    const backoffPeriods = [1, 5, 15];
+    for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
+        const res = await client.put(url, payload);
+        core.debug('response status: ' + res.message.statusCode);
+        const statusCode = res.message.statusCode;
+        if (statusCode === 202) {
+            core.info("Successfully uploaded results");
+            return;
+        }
+        const requestID = res.message.headers["x-github-request-id"];
+        // On any other status code that's not 5xx mark the upload as failed
+        if (!statusCode || statusCode < 500 || statusCode >= 600) {
+            core.setFailed('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+        // On a 5xx status code we may retry the request
+        if (attempt < backoffPeriods.length) {
+            // Log the failure as a warning but don't mark the action as failed yet
+            core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
+                ') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
+                ' seconds: (' + statusCode + ') ' + await res.readBody());
+            // Sleep for the backoff period
+            await new Promise(r => setTimeout(r, backoffPeriods[attempt] * 1000));
+            continue;
+        }
+        else {
+            // If the upload fails with 5xx then we assume it is a temporary problem
+            // and not an error that the user has caused or can fix.
+            // We avoid marking the job as failed to avoid breaking CI workflows.
+            core.error('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+    }
+}
 // Uploads a single sarif file or a directory of sarif files
 // depending on what the path happens to refer to.
 async function upload(input) {
@@ -112,26 +158,8 @@ async function uploadFiles(sarifFiles) {
             "started_at": startedAt,
             "tool_names": toolNames,
         });
-        core.info('Uploading results');
-        const githubToken = core.getInput('token');
-        const ph = new auth.BearerCredentialHandler(githubToken);
-        const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
-        const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
-        const res = await client.put(url, payload);
-        const requestID = res.message.headers["x-github-request-id"];
-        core.debug('response status: ' + res.message.statusCode);
-        if (res.message.statusCode === 500) {
-            // If the upload fails with 500 then we assume it is a temporary problem
-            // with turbo-scan and not an error that the user has caused or can fix.
-            // We avoid marking the job as failed to avoid breaking CI workflows.
-            core.error('Upload failed (' + requestID + '): ' + await res.readBody());
-        }
-        else if (res.message.statusCode !== 202) {
-            core.setFailed('Upload failed (' + requestID + '): ' + await res.readBody());
-        }
-        else {
-            core.info("Successfully uploaded results");
-        }
+        // Make the upload
+        await uploadPayload(payload);
         // Mark that we have made an upload
         fs.writeFileSync(sentinelFile, '');
     }

package-lock.json

@@ -15,9 +15,9 @@
       "integrity": "sha512-nvFkxwiicvpzNiCBF4wFBDfnBvi7xp/as7LE1hBxBxKG2L29+gkIPBiLKMVORL+Hg3JNf07AKRfl0V5djoypjQ=="
     },
     "@actions/http-client": {
-      "version": "1.0.4",
-      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.4.tgz",
-      "integrity": "sha512-6EzXhqapKKtYr21ZnFQVBYwfrYPKPCivuSkUN/66/BDakkH2EPjUZH8tZ3MgHdI+gQIdcsY0ybbxw9ZEOmJB6g==",
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-1.0.8.tgz",
+      "integrity": "sha512-G4JjJ6f9Hb3Zvejj+ewLLKLf99ZC+9v+yCxoYf9vSyH+WkzPLB2LuUtRMGNkooMqdugGBFStIKXOuvH1W+EctA==",
       "requires": {
         "tunnel": "0.0.6"
       },

package.json

@@ -12,7 +12,7 @@
   "dependencies": {
     "@actions/core": "^1.0.0",
     "@actions/exec": "^1.0.1",
-    "@actions/http-client": "^1.0.4",
+    "@actions/http-client": "^1.0.8",
     "@actions/io": "^1.0.1",
     "@actions/tool-cache": "^1.1.2",
     "@octokit/rest": "^17.1.0",

src/config-utils.ts

@@ -17,6 +17,7 @@ export class ExternalQuery {
 
 export class Config {
     public name = "";
+    public disableDefaultQueries = false;
     public additionalQueries: string[] = [];
     public externalQueries: ExternalQuery[] = [];
     public pathsIgnore: string[] = [];
@@ -81,6 +82,10 @@ function initConfig(): Config {
             config.name = parsedYAML.name;
         }
 
+        if (parsedYAML['disable-default-queries'] && typeof parsedYAML['disable-default-queries'] === "boolean") {
+            config.disableDefaultQueries = parsedYAML['disable-default-queries'];
+        }
+
         const queries = parsedYAML.queries;
         if (queries && queries instanceof Array) {
             queries.forEach(query => {

src/finalize-db.ts

@@ -82,13 +82,13 @@ async function resolveQueryLanguages(codeqlCmd: string, config: configUtils.Conf
     const noDeclaredLanguage = resolveQueriesOutputObject.noDeclaredLanguage;
     const noDeclaredLanguageQueries = Object.keys(noDeclaredLanguage);
     if (noDeclaredLanguageQueries.length !== 0) {
-      core.warning('Some queries do not declare a language:\n' + noDeclaredLanguageQueries.join('\n'));
+      throw new Error('Some queries do not declare a language, their qlpack.yml file is missing or is invalid');
     }
 
     const multipleDeclaredLanguages = resolveQueriesOutputObject.multipleDeclaredLanguages;
     const multipleDeclaredLanguagesQueries = Object.keys(multipleDeclaredLanguages);
     if (multipleDeclaredLanguagesQueries.length !== 0) {
-      core.warning('Some queries declare multiple languages:\n' + multipleDeclaredLanguagesQueries.join('\n'));
+      throw new Error('Some queries declare multiple languages, their qlpack.yml file is missing or is invalid');
     }
   }
 
@@ -102,7 +102,12 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
   for (let database of fs.readdirSync(databaseFolder)) {
     core.startGroup('Analyzing ' + database);
 
-    const additionalQueries = queriesPerLanguage[database] || [];
+    const queries: string[] = [];
+    if (!config.disableDefaultQueries) {
+      queries.push(database + '-code-scanning.qls');
+    }
+    queries.push(...(queriesPerLanguage[database] || []));
+
     const sarifFile = path.join(sarifFolder, database + '.sarif');
 
     await exec.exec(codeqlCmd, [
@@ -112,8 +117,7 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
       '--format=sarif-latest',
       '--output=' + sarifFile,
       '--no-sarif-add-snippets',
-      database + '-code-scanning.qls',
-      ...additionalQueries,
+      ...queries
     ]);
 
     core.debug('SARIF results for database ' + database + ' created at "' + sarifFile + '"');

src/upload-lib.ts

@@ -47,6 +47,61 @@ export function combineSarifFiles(sarifFiles: string[]): string {
     return JSON.stringify(combinedSarif);
 }
 
+// Upload the given payload.
+// If the request fails then this will retry a small number of times.
+async function uploadPayload(payload) {
+    core.info('Uploading results');
+
+    const githubToken = core.getInput('token');
+    const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
+    const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
+    const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
+
+    // Make up to 4 attempts to upload, and sleep for these
+    // number of seconds between each attempt.
+    // We don't want to backoff too much to avoid wasting action
+    // minutes, but just waiting a little bit could maybe help.
+    const backoffPeriods = [1, 5, 15];
+
+    for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
+
+        const res: http.HttpClientResponse = await client.put(url, payload);
+        core.debug('response status: ' + res.message.statusCode);
+
+        const statusCode = res.message.statusCode;
+        if (statusCode === 202) {
+            core.info("Successfully uploaded results");
+            return;
+        }
+
+        const requestID = res.message.headers["x-github-request-id"];
+
+        // On any other status code that's not 5xx mark the upload as failed
+        if (!statusCode || statusCode < 500 || statusCode >= 600) {
+            core.setFailed('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+
+        // On a 5xx status code we may retry the request
+        if (attempt < backoffPeriods.length) {
+            // Log the failure as a warning but don't mark the action as failed yet
+            core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
+              ') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
+              ' seconds: (' + statusCode + ') ' + await res.readBody());
+            // Sleep for the backoff period
+            await new Promise(r => setTimeout(r, backoffPeriods[attempt] * 1000));
+            continue;
+
+        } else {
+            // If the upload fails with 5xx then we assume it is a temporary problem
+            // and not an error that the user has caused or can fix.
+            // We avoid marking the job as failed to avoid breaking CI workflows.
+            core.error('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+    }
+}
+
 // Uploads a single sarif file or a directory of sarif files
 // depending on what the path happens to refer to.
 export async function upload(input: string) {
@@ -112,25 +167,8 @@ async function uploadFiles(sarifFiles: string[]) {
             "tool_names": toolNames,
         });
 
-        core.info('Uploading results');
-        const githubToken = core.getInput('token');
-        const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
-        const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
-        const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
-        const res: http.HttpClientResponse = await client.put(url, payload);
-        const requestID = res.message.headers["x-github-request-id"];
-
-        core.debug('response status: ' + res.message.statusCode);
-        if (res.message.statusCode === 500) {
-            // If the upload fails with 500 then we assume it is a temporary problem
-            // with turbo-scan and not an error that the user has caused or can fix.
-            // We avoid marking the job as failed to avoid breaking CI workflows.
-            core.error('Upload failed (' + requestID + '): ' + await res.readBody());
-        } else if (res.message.statusCode !== 202) {
-            core.setFailed('Upload failed (' + requestID + '): ' + await res.readBody());
-        } else {
-            core.info("Successfully uploaded results");
-        }
+        // Make the upload
+        await uploadPayload(payload);
 
         // Mark that we have made an upload
         fs.writeFileSync(sentinelFile, '');