Merge branch 'master' into upload_twice

analyze/action.yml

@@ -12,6 +12,9 @@ inputs:
     description: Upload the SARIF file
     required: false
     default: true
+  ram:
+    description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
+    required: false
   token:
     default: ${{ github.token }}
   matrix:

lib/finalize-db.js

@@ -11,12 +11,30 @@ const core = __importStar(require("@actions/core"));
 const exec = __importStar(require("@actions/exec"));
 const io = __importStar(require("@actions/io"));
 const fs = __importStar(require("fs"));
+const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const configUtils = __importStar(require("./config-utils"));
 const externalQueries = __importStar(require("./external-queries"));
 const sharedEnv = __importStar(require("./shared-environment"));
 const upload_lib = __importStar(require("./upload-lib"));
 const util = __importStar(require("./util"));
+function getMemoryFlag() {
+    let memoryToUseMegaBytes;
+    const memoryToUseString = core.getInput("ram");
+    if (memoryToUseString) {
+        memoryToUseMegaBytes = Number(memoryToUseString);
+        if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
+            throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
+        }
+    }
+    else {
+        const totalMemoryBytes = os.totalmem();
+        const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
+        const systemReservedMemoryMegaBytes = 256;
+        memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
+    }
+    return "--ram=" + Math.floor(memoryToUseMegaBytes);
+}
 async function createdDBForScannedLanguages(codeqlCmd, databaseFolder) {
     const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
     if (scannedLanguages) {
@@ -97,6 +115,7 @@ async function runQueries(codeqlCmd, databaseFolder, sarifFolder, config) {
         await exec.exec(codeqlCmd, [
             'database',
             'analyze',
+            getMemoryFlag(),
             path.join(databaseFolder, database),
             '--format=sarif-latest',
             '--output=' + sarifFile,

lib/upload-lib.js

@@ -112,6 +112,15 @@ async function upload(input) {
     }
 }
 exports.upload = upload;
+// Counts the number of results in the given SARIF file
+function countResultsInSarif(sarif) {
+    let numResults = 0;
+    for (const run of JSON.parse(sarif).runs) {
+        numResults += run.results.length;
+    }
+    return numResults;
+}
+exports.countResultsInSarif = countResultsInSarif;
 // Uploads the given set of sarif files.
 // Returns true iff the upload occurred and succeeded
 async function uploadFiles(sarifFiles) {
@@ -158,6 +167,10 @@ async function uploadFiles(sarifFiles) {
             "started_at": startedAt,
             "tool_names": toolNames,
         });
+        // Log some useful debug info about the info
+        core.debug("Raw upload size: " + sarifPayload.length + " bytes");
+        core.debug("Base64 zipped upload size: " + zipped_sarif.length + " bytes");
+        core.debug("Number of results in upload: " + countResultsInSarif(sarifPayload));
         // Make the upload
         succeeded = await uploadPayload(payload);
     }

src/finalize-db.ts

@@ -2,6 +2,7 @@ import * as core from '@actions/core';
 import * as exec from '@actions/exec';
 import * as io from '@actions/io';
 import * as fs from 'fs';
+import * as os from 'os';
 import * as path from 'path';
 
 import * as configUtils from './config-utils';
@@ -10,6 +11,23 @@ import * as sharedEnv from './shared-environment';
 import * as upload_lib from './upload-lib';
 import * as util from './util';
 
+function getMemoryFlag(): string {
+  let memoryToUseMegaBytes: number;
+  const memoryToUseString = core.getInput("ram");
+  if (memoryToUseString) {
+    memoryToUseMegaBytes = Number(memoryToUseString);
+    if (Number.isNaN(memoryToUseMegaBytes) || memoryToUseMegaBytes <= 0) {
+      throw new Error("Invalid RAM setting \"" + memoryToUseString + "\", specified.");
+    }
+  } else {
+    const totalMemoryBytes = os.totalmem();
+    const totalMemoryMegaBytes = totalMemoryBytes / (1024 * 1024);
+    const systemReservedMemoryMegaBytes = 256;
+    memoryToUseMegaBytes = totalMemoryMegaBytes - systemReservedMemoryMegaBytes;
+  }
+  return "--ram=" + Math.floor(memoryToUseMegaBytes);
+}
+
 async function createdDBForScannedLanguages(codeqlCmd: string, databaseFolder: string) {
   const scannedLanguages = process.env[sharedEnv.CODEQL_ACTION_SCANNED_LANGUAGES];
   if (scannedLanguages) {
@@ -113,6 +131,7 @@ async function runQueries(codeqlCmd: string, databaseFolder: string, sarifFolder
     await exec.exec(codeqlCmd, [
       'database',
       'analyze',
+      getMemoryFlag(),
       path.join(databaseFolder, database),
       '--format=sarif-latest',
       '--output=' + sarifFile,

src/upload-lib.ts

@@ -114,6 +114,15 @@ export async function upload(input: string): Promise<boolean> {
     }
 }
 
+// Counts the number of results in the given SARIF file
+export function countResultsInSarif(sarif: string): number {
+    let numResults = 0;
+    for (const run of JSON.parse(sarif).runs) {
+        numResults += run.results.length;
+    }
+    return numResults;
+}
+
 // Uploads the given set of sarif files.
 // Returns true iff the upload occurred and succeeded
 async function uploadFiles(sarifFiles: string[]): Promise<boolean> {
@@ -168,6 +177,11 @@ async function uploadFiles(sarifFiles: string[]): Promise<boolean> {
             "tool_names": toolNames,
         });
 
+        // Log some useful debug info about the info
+        core.debug("Raw upload size: " + sarifPayload.length + " bytes");
+        core.debug("Base64 zipped upload size: " + zipped_sarif.length + " bytes");
+        core.debug("Number of results in upload: " + countResultsInSarif(sarifPayload));
+
         // Make the upload
         succeeded = await uploadPayload(payload);