Merge branch 'main' into henrymercer/disk-telemetry

.github/actions/setup-swift/action.yml

@@ -1,5 +1,5 @@
-name: "Set up Swift"
-description: Sets up an appropriate Swift version if supported on this platform.
+name: "Set up Swift on Linux"
+description: Sets up an appropriate Swift version on Linux.
 inputs:
   codeql-path:
     description: Path to the CodeQL CLI executable.
@@ -9,21 +9,16 @@ runs:
   steps:
     - name: Get Swift version
       id: get_swift_version
-      if: runner.os != 'Windows'
+      if: runner.os == 'Linux'
       shell: bash
       env: 
         CODEQL_PATH: ${{ inputs.codeql-path }}
       run: |
-        if [[ $RUNNER_OS = "macOS" ]]; then
-          PLATFORM="osx64"
-        else # We do not run this step on Windows.
-          PLATFORM="linux64"
-        fi 
         SWIFT_EXTRACTOR_DIR="$("$CODEQL_PATH" resolve languages --format json | jq -r '.swift[0]')"
         if [ $SWIFT_EXTRACTOR_DIR = "null" ]; then
           VERSION="null"
         else
-          VERSION="$("$SWIFT_EXTRACTOR_DIR/tools/$PLATFORM/extractor" --version | awk '/version/ { print $3 }')"
+          VERSION="$("$SWIFT_EXTRACTOR_DIR/tools/linux64/extractor" --version | awk '/version/ { print $3 }')"
           # Specify 5.x.0, otherwise setup Action will default to latest minor version. 
           if [ $VERSION = "5.7" ]; then
             VERSION="5.7.0"
@@ -37,6 +32,6 @@ runs:
         echo "version=$VERSION" | tee -a $GITHUB_OUTPUT
 
     - uses: swift-actions/setup-swift@65540b95f51493d65f5e59e97dcef9629ddf11bf # Please update the corresponding SHA in the CLI's CodeQL Action Integration Test.
-      if: runner.os != 'Windows' && steps.get_swift_version.outputs.version != 'null'
+      if: runner.os == 'Linux' && steps.get_swift_version.outputs.version != 'null'
       with:
         swift-version: "${{ steps.get_swift_version.outputs.version }}"

.github/workflows/__export-file-baseline-information.yml

@@ -63,7 +63,7 @@ jobs:
         CODEQL_FILE_BASELINE_INFORMATION: true
     - uses: ./../action/.github/actions/setup-swift
       with:
-        codeql-path: ${{steps.init.outputs.codeql-path}}
+        codeql-path: ${{ steps.init.outputs.codeql-path }}
     - name: Build code
       shell: bash
       run: ./build.sh

lib/codeql.js

@@ -23,7 +23,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getGeneratedCodeScanningConfigPath = exports.getTrapCachingExtractorConfigArgsForLang = exports.getTrapCachingExtractorConfigArgs = exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_RESOLVE_ENVIRONMENT = exports.CODEQL_VERSION_BETTER_NO_CODE_ERROR_MESSAGE = exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = exports.CODEQL_VERSION_EXPORT_CODE_SCANNING_CONFIG = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
+exports.getGeneratedCodeScanningConfigPath = exports.getTrapCachingExtractorConfigArgsForLang = exports.getTrapCachingExtractorConfigArgs = exports.getExtraOptions = exports.getCodeQLForCmd = exports.getCodeQLForTesting = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.setupCodeQL = exports.CODEQL_VERSION_RESOLVE_ENVIRONMENT = exports.CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED = exports.CODEQL_VERSION_BETTER_NO_CODE_ERROR_MESSAGE = exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = exports.CODEQL_VERSION_EXPORT_CODE_SCANNING_CONFIG = exports.CODEQL_VERSION_SECURITY_EXPERIMENTAL_SUITE = exports.CODEQL_VERSION_BETTER_RESOLVE_LANGUAGES = exports.CODEQL_VERSION_GHES_PACK_DOWNLOAD = exports.CommandInvocationError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const core = __importStar(require("@actions/core"));
@@ -100,6 +100,10 @@ exports.CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
  * determines that no code has been found.
  */
 exports.CODEQL_VERSION_BETTER_NO_CODE_ERROR_MESSAGE = "2.12.4";
+/**
+ * Versions 2.13.1+ of the CodeQL CLI fix a bug where diagnostics export could produce invalid SARIF.
+ */
+exports.CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED = "2.13.1";
 /**
  * Versions 2.13.4+ of the CodeQL CLI support the `resolve build-environment` command.
  */
@@ -449,8 +453,8 @@ async function getCodeQLForCmd(cmd, checkVersion) {
         },
         async databaseInterpretResults(databasePath, querySuitePaths, sarifFile, addSnippetsFlag, threadsFlag, verbosityFlag, automationDetailsId, config, features, logger) {
             const shouldExportDiagnostics = await features.getValue(feature_flags_1.Feature.ExportDiagnosticsEnabled, this);
-            // Update this to take into account the CodeQL version when we have a version with the fix.
-            const shouldWorkaroundInvalidNotifications = shouldExportDiagnostics;
+            const shouldWorkaroundInvalidNotifications = shouldExportDiagnostics &&
+                !(await isDiagnosticsExportInvalidSarifFixed(this));
             const codeqlOutputFile = shouldWorkaroundInvalidNotifications
                 ? path.join(config.tempDir, "codeql-intermediate-results.sarif")
                 : sarifFile;
@@ -576,8 +580,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             await new toolrunner.ToolRunner(cmd, args).exec();
         },
         async databaseExportDiagnostics(databasePath, sarifFile, automationDetailsId, tempDir, logger) {
-            // Update this to take into account the CodeQL version when we have a version with the fix.
-            const shouldWorkaroundInvalidNotifications = true;
+            const shouldWorkaroundInvalidNotifications = !(await isDiagnosticsExportInvalidSarifFixed(this));
             const codeqlOutputFile = shouldWorkaroundInvalidNotifications
                 ? path.join(tempDir, "codeql-intermediate-results.sarif")
                 : sarifFile;
@@ -949,4 +952,7 @@ function isNoCodeFoundError(e) {
     const javascriptNoCodeFoundWarning = "No JavaScript or TypeScript code found.";
     return e.exitCode === 32 || e.error.includes(javascriptNoCodeFoundWarning);
 }
+async function isDiagnosticsExportInvalidSarifFixed(codeql) {
+    return await util.codeQlVersionAbove(codeql, exports.CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED);
+}
 //# sourceMappingURL=codeql.js.map

lib/util.js

@@ -110,9 +110,9 @@ function getSystemReservedMemoryMegaBytes(totalMemoryMegaBytes, platform, isScal
     // Windows needs more memory for OS processes.
     const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
     if (isScalingReservedRamEnabled) {
-        // Reserve an additional 2.5% of the amount of memory above 8 GB, since the amount used by
-        // the kernel for page tables scales with the size of physical memory.
-        const scaledAmount = 0.025 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
+        // Reserve an additional 5% of the amount of memory above 8 GB, since the amount used by the
+        // kernel for page tables scales with the size of physical memory.
+        const scaledAmount = 0.05 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
         return fixedAmount + scaledAmount;
     }
     else {

lib/util.test.js

@@ -73,14 +73,14 @@ const GET_MEMORY_FLAG_TESTS = [
         totalMemoryMb: 64 * 1024,
         platform: "linux",
         expectedMemoryValue: 63 * 1024,
-        expectedMemoryValueWithScaling: 63078, // Math.floor(1024 * (64 - 1 - 0.025 * (64 - 8)))
+        expectedMemoryValueWithScaling: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
     },
     {
         input: undefined,
         totalMemoryMb: 64 * 1024,
         platform: "win32",
         expectedMemoryValue: 62.5 * 1024,
-        expectedMemoryValueWithScaling: 62566, // Math.floor(1024 * (64 - 1.5 - 0.025 * (64 - 8)))
+        expectedMemoryValueWithScaling: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
     },
 ];
 for (const { input, totalMemoryMb, platform, expectedMemoryValue, expectedMemoryValueWithScaling, } of GET_MEMORY_FLAG_TESTS) {

pr-checks/checks/export-file-baseline-information.yml

@@ -11,7 +11,7 @@ steps:
       CODEQL_FILE_BASELINE_INFORMATION: true
   - uses: ./../action/.github/actions/setup-swift
     with:
-      codeql-path: ${{steps.init.outputs.codeql-path}}
+      codeql-path: ${{ steps.init.outputs.codeql-path }}
   - name: Build code
     shell: bash
     run: ./build.sh

src/codeql.ts

@@ -315,6 +315,11 @@ export const CODEQL_VERSION_INIT_WITH_QLCONFIG = "2.12.4";
  */
 export const CODEQL_VERSION_BETTER_NO_CODE_ERROR_MESSAGE = "2.12.4";
 
+/**
+ * Versions 2.13.1+ of the CodeQL CLI fix a bug where diagnostics export could produce invalid SARIF.
+ */
+export const CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED = "2.13.1";
+
 /**
  * Versions 2.13.4+ of the CodeQL CLI support the `resolve build-environment` command.
  */
@@ -814,8 +819,9 @@ export async function getCodeQLForCmd(
         Feature.ExportDiagnosticsEnabled,
         this,
       );
-      // Update this to take into account the CodeQL version when we have a version with the fix.
-      const shouldWorkaroundInvalidNotifications = shouldExportDiagnostics;
+      const shouldWorkaroundInvalidNotifications =
+        shouldExportDiagnostics &&
+        !(await isDiagnosticsExportInvalidSarifFixed(this));
       const codeqlOutputFile = shouldWorkaroundInvalidNotifications
         ? path.join(config.tempDir, "codeql-intermediate-results.sarif")
         : sarifFile;
@@ -970,8 +976,8 @@ export async function getCodeQLForCmd(
       tempDir: string,
       logger: Logger,
     ): Promise<void> {
-      // Update this to take into account the CodeQL version when we have a version with the fix.
-      const shouldWorkaroundInvalidNotifications = true;
+      const shouldWorkaroundInvalidNotifications =
+        !(await isDiagnosticsExportInvalidSarifFixed(this));
       const codeqlOutputFile = shouldWorkaroundInvalidNotifications
         ? path.join(tempDir, "codeql-intermediate-results.sarif")
         : sarifFile;
@@ -1413,3 +1419,12 @@ function isNoCodeFoundError(e: CommandInvocationError): boolean {
     "No JavaScript or TypeScript code found.";
   return e.exitCode === 32 || e.error.includes(javascriptNoCodeFoundWarning);
 }
+
+async function isDiagnosticsExportInvalidSarifFixed(
+  codeql: CodeQL,
+): Promise<boolean> {
+  return await util.codeQlVersionAbove(
+    codeql,
+    CODEQL_VERSION_DIAGNOSTICS_EXPORT_FIXED,
+  );
+}

src/util.test.ts

@@ -53,14 +53,14 @@ const GET_MEMORY_FLAG_TESTS = [
     totalMemoryMb: 64 * 1024,
     platform: "linux",
     expectedMemoryValue: 63 * 1024,
-    expectedMemoryValueWithScaling: 63078, // Math.floor(1024 * (64 - 1 - 0.025 * (64 - 8)))
+    expectedMemoryValueWithScaling: 61644, // Math.floor(1024 * (64 - 1 - 0.05 * (64 - 8)))
   },
   {
     input: undefined,
     totalMemoryMb: 64 * 1024,
     platform: "win32",
     expectedMemoryValue: 62.5 * 1024,
-    expectedMemoryValueWithScaling: 62566, // Math.floor(1024 * (64 - 1.5 - 0.025 * (64 - 8)))
+    expectedMemoryValueWithScaling: 61132, // Math.floor(1024 * (64 - 1.5 - 0.05 * (64 - 8)))
   },
 ];
 

src/util.ts

@@ -161,9 +161,9 @@ function getSystemReservedMemoryMegaBytes(
   const fixedAmount = 1024 * (platform === "win32" ? 1.5 : 1);
 
   if (isScalingReservedRamEnabled) {
-    // Reserve an additional 2.5% of the amount of memory above 8 GB, since the amount used by
-    // the kernel for page tables scales with the size of physical memory.
-    const scaledAmount = 0.025 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
+    // Reserve an additional 5% of the amount of memory above 8 GB, since the amount used by the
+    // kernel for page tables scales with the size of physical memory.
+    const scaledAmount = 0.05 * Math.max(totalMemoryMegaBytes - 8 * 1024, 0);
     return fixedAmount + scaledAmount;
   } else {
     return fixedAmount;