Add RAM and threads options to init action

.github/workflows/__extractor-ram-threads.yml

@@ -0,0 +1,63 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: PR Check - Extractor ram and threads options test
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - v1
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  extractor-ram-threads:
+    strategy:
+      matrix:
+        version: [latest]
+        os: [ubuntu-latest]
+    name: Extractor ram and threads options test
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: ./../action/init
+      with:
+        languages: java
+        ram: 230
+        threads: 1
+    - name: Assert Results
+      shell: bash
+      run: |
+        if [ "${CODEQL_RAM}" != "230" ]; then
+          echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
+          exit 1
+        fi
+        if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
+          echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
+          exit 1
+        fi
+        if [ "${CODEQL_THREADS}" != "1" ]; then
+          echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
+          exit 1
+        fi
+        if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
+          echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
+          exit 1
+        fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/pr-checks.yml

@@ -393,3 +393,42 @@ jobs:
           # Deliberately don't use TEST_MODE here. This is specifically testing
           # the compatibility with the API.
           runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  runner-extractor-ram-threads-options:
+    name: Runner ubuntu extractor RAM and threads options
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build runner
+        run: |
+          cd runner
+          npm install
+          npm run build-runner
+
+      - name: Run init
+        run: |
+          runner/dist/codeql-runner-linux init --ram=230 --threads=1 --repository $GITHUB_REPOSITORY --languages java --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+      - name: Assert Results
+        shell: bash
+        run: |
+          . ./codeql-runner/codeql-env.sh
+          if [ "${CODEQL_RAM}" != "230" ]; then
+            echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
+            exit 1
+          fi
+          if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
+            echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
+            exit 1
+          fi
+          if [ "${CODEQL_THREADS}" != "1" ]; then
+            echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
+            exit 1
+          fi
+          if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
+            echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
+            exit 1
+          fi

CHANGELOG.md

@@ -2,7 +2,7 @@
 
 ## [UNRELEASED]
 
-No user facing changes.
+- The `init` step of the Action now supports `ram` and `threads` inputs to limit resource use of CodeQL extractors. These inputs also serve as defaults to the subsequent `analyze` step, which finalizes the database and executes queries. [#738](https://github.com/github/codeql-action/pull/738)
 
 ## 1.0.21 - 28 Oct 2021
 

analyze/action.yml

@@ -18,7 +18,12 @@ inputs:
     required: false
     default: "brutal"
   ram:
-    description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
+    description: >-
+      The amount of memory in MB that can be used by CodeQL for database finalization and query execution.
+      By default, this action will use the same amount of memory as previously set in the "init" action.
+      If the "init" action also does not have an explicit "ram" input, this action will use most of the
+      memory available in the system (which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows,
+      and 13GB for macOS).
     required: false
   add-snippets:
     description: Specify whether or not to add code snippets to the output sarif file.
@@ -29,7 +34,12 @@ inputs:
     required: false
     default: "false"
   threads:
-    description: The number of threads to be used by CodeQL.
+    description: >-
+      The number of threads that can be used by CodeQL for database finalization and query execution.
+      By default, this action will use the same number of threads as previously set in the "init" action.
+      If the "init" action also does not have an explicit "threads" input, this action will use all the
+      hardware threads available in the system (which for GitHub-hosted runners is 2 for Linux and Windows
+      and 3 for macOS).
     required: false
   checkout_path:
     description: "The path at which the analyzed repository was checked out. Used to relativize any absolute paths in the uploaded SARIF file."

init/action.yml

@@ -41,6 +41,20 @@ inputs:
   source-root:
     description: Path of the root source code directory, relative to $GITHUB_WORKSPACE.
     required: false
+  ram:
+    description: >-
+      The amount of memory in MB that can be used by CodeQL extractors.
+      By default, CodeQL extractors will use most of the memory available in the system
+      (which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows, and 13GB for macOS).
+      This input also sets the amount of memory that can later be used by the "analyze" action.
+    required: false
+  threads:
+    description: >-
+      The number of threads that can be used by CodeQL extractors.
+      By default, CodeQL extractors will use all the hardware threads available in the system
+      (which for GitHub-hosted runners is 2 for Linux and Windows and 3 for macOS).
+      This input also sets the number of threads that can later be used by the "analyze" action.
+    required: false
 outputs:
   codeql-path:
     description: The path of the CodeQL binary used for analysis

lib/analyze-action-env.test.js

@@ -0,0 +1,75 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const ava_1 = __importDefault(require("ava"));
+const sinon = __importStar(require("sinon"));
+const actionsUtil = __importStar(require("./actions-util"));
+const analyze = __importStar(require("./analyze"));
+const configUtils = __importStar(require("./config-utils"));
+const testing_utils_1 = require("./testing-utils");
+const util = __importStar(require("./util"));
+(0, testing_utils_1.setupTests)(ava_1.default);
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+(0, ava_1.default)("analyze action with RAM & threads from environment variables", async (t) => {
+    await util.withTmpDir(async (tmpDir) => {
+        process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+        process.env["GITHUB_REPOSITORY"] = "fake/repository";
+        sinon
+            .stub(actionsUtil, "createStatusReportBase")
+            .resolves({});
+        sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+        sinon.stub(configUtils, "getConfig").resolves({
+            languages: [],
+        });
+        const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+        requiredInputStub.withArgs("token").returns("fake-token");
+        requiredInputStub.withArgs("upload-database").returns("false");
+        const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+        optionalInputStub.withArgs("cleanup-level").returns("none");
+        (0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
+        // When there are no action inputs for RAM and threads, the action uses
+        // environment variables (passed down from the init action) to set RAM and
+        // threads usage.
+        process.env["CODEQL_THREADS"] = "-1";
+        process.env["CODEQL_RAM"] = "4992";
+        const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+        const runQueriesStub = sinon.stub(analyze, "runQueries");
+        const analyzeAction = require("./analyze-action");
+        // When analyze-action.ts loads, it runs an async function from the top
+        // level but does not wait for it to finish. To ensure that calls to
+        // runFinalize and runQueries are correctly captured by spies, we explicitly
+        // wait for the action promise to complete before starting verification.
+        await analyzeAction.runPromise;
+        t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+        t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
+        t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+        t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
+    });
+});
+//# sourceMappingURL=analyze-action-env.test.js.map

lib/analyze-action-env.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/analyze-action-input.test.js

@@ -0,0 +1,75 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const ava_1 = __importDefault(require("ava"));
+const sinon = __importStar(require("sinon"));
+const actionsUtil = __importStar(require("./actions-util"));
+const analyze = __importStar(require("./analyze"));
+const configUtils = __importStar(require("./config-utils"));
+const testing_utils_1 = require("./testing-utils");
+const util = __importStar(require("./util"));
+(0, testing_utils_1.setupTests)(ava_1.default);
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+(0, ava_1.default)("analyze action with RAM & threads from action inputs", async (t) => {
+    await util.withTmpDir(async (tmpDir) => {
+        process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+        process.env["GITHUB_REPOSITORY"] = "fake/repository";
+        sinon
+            .stub(actionsUtil, "createStatusReportBase")
+            .resolves({});
+        sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+        sinon.stub(configUtils, "getConfig").resolves({
+            languages: [],
+        });
+        const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+        requiredInputStub.withArgs("token").returns("fake-token");
+        requiredInputStub.withArgs("upload-database").returns("false");
+        const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+        optionalInputStub.withArgs("cleanup-level").returns("none");
+        (0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
+        process.env["CODEQL_THREADS"] = "1";
+        process.env["CODEQL_RAM"] = "4992";
+        // Action inputs have precedence over environment variables.
+        optionalInputStub.withArgs("threads").returns("-1");
+        optionalInputStub.withArgs("ram").returns("3012");
+        const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+        const runQueriesStub = sinon.stub(analyze, "runQueries");
+        const analyzeAction = require("./analyze-action");
+        // When analyze-action.ts loads, it runs an async function from the top
+        // level but does not wait for it to finish. To ensure that calls to
+        // runFinalize and runQueries are correctly captured by spies, we explicitly
+        // wait for the action promise to complete before starting verification.
+        await analyzeAction.runPromise;
+        t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+        t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
+        t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+        t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
+    });
+});
+//# sourceMappingURL=analyze-action-input.test.js.map

lib/analyze-action-input.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/analyze-action.js

@@ -19,6 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.runPromise = exports.sendStatusReport = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const core = __importStar(require("@actions/core"));
@@ -44,6 +45,7 @@ async function sendStatusReport(startedAt, stats, error) {
     };
     await actionsUtil.sendStatusReport(statusReport);
 }
+exports.sendStatusReport = sendStatusReport;
 async function run() {
     const startedAt = new Date();
     let uploadStats = undefined;
@@ -65,8 +67,8 @@ async function run() {
             url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
         };
         const outputDir = actionsUtil.getRequiredInput("output");
-        const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger);
+        const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
-        const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram"));
+        const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
         await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
@@ -134,9 +136,10 @@ async function run() {
         await sendStatusReport(startedAt, undefined);
     }
 }
+exports.runPromise = run();
 async function runWrapper() {
     try {
-        await run();
+        await exports.runPromise;
     }
     catch (error) {
         core.setFailed(`analyze action failed: ${error}`);

lib/init-action.js

@@ -113,9 +113,14 @@ async function run() {
             core.exportVariable("GOFLAGS", goFlags);
             core.warning("Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action.");
         }
-        // Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
+        // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
-        const codeqlRam = process.env["CODEQL_RAM"] || "6500";
+        // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
-        core.exportVariable("CODEQL_RAM", codeqlRam);
+        // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+        // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+        // for details.
+        core.exportVariable("CODEQL_RAM", process.env["CODEQL_RAM"] ||
+            (0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram")).toString());
+        core.exportVariable("CODEQL_THREADS", (0, util_1.getThreadsFlagValue)((0, actions_util_1.getOptionalInput)("threads"), logger).toString());
         const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
         const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", undefined);
         if (tracerConfig !== undefined) {

lib/runner.js

@@ -54,11 +54,14 @@ function getToolsDir(userInput) {
     return toolsDir;
 }
 const codeqlEnvJsonFilename = "codeql-env.json";
+function loadTracerEnvironment(config) {
+    const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
+    return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
+}
 // Imports the environment from codeqlEnvJsonFilename if not already present
 function importTracerEnvironment(config) {
     if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
-        const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
+        const env = loadTracerEnvironment(config);
-        const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
         for (const key of Object.keys(env)) {
             process.env[key] = env[key];
         }
@@ -116,6 +119,12 @@ program
     .option("--debug", "Print more verbose output", false)
     .option("--trace-process-name <string>", "(Advanced, windows-only) Inject a windows tracer of this process into a process with the given process name.")
     .option("--trace-process-level <number>", "(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up.")
+    .option("--ram <number>", "The amount of memory in MB that can be used by CodeQL extractors. " +
+    "By default, CodeQL extractors will use most of the memory available in the system. " +
+    'This input also sets the amount of memory that can later be used by the "analyze" command.')
+    .option("--threads <number>", "The number of threads that can be used by CodeQL extractors. " +
+    "By default, CodeQL extractors will use all the hardware threads available in the system. " +
+    'This input also sets the number of threads that can later be used by the "analyze" command.')
     .action(async (cmd) => {
     const logger = (0, logging_1.getRunnerLogger)(cmd.debug);
     try {
@@ -134,6 +143,13 @@ program
         };
         const gitHubVersion = await (0, util_1.getGitHubVersion)(apiDetails);
         (0, util_1.checkGitHubVersionInRange)(gitHubVersion, logger, util_1.Mode.runner);
+        // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
+        // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
+        // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+        // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+        // for details.
+        process.env["CODEQL_RAM"] = (0, util_1.getMemoryFlagValue)(cmd.ram).toString();
+        process.env["CODEQL_THREADS"] = (0, util_1.getThreadsFlagValue)(cmd.threads, logger).toString();
         let codeql;
         if (cmd.codeqlPath !== undefined) {
             codeql = await (0, codeql_1.getCodeQL)(cmd.codeqlPath);
@@ -240,10 +256,15 @@ program
     .option("--checkout-path <path>", "Checkout path. Default is the current working directory.")
     .option("--no-upload", "Do not upload results after analysis.")
     .option("--output-dir <dir>", "Directory to output SARIF files to. Default is in the temp directory.")
-    .option("--ram <ram>", "Amount of memory to use when running queries. Default is to use all available memory.")
+    .option("--ram <ram>", "The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
+    'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
+    'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
+    "memory available in the system.")
     .option("--no-add-snippets", "Specify whether to include code snippets in the sarif output.")
-    .option("--threads <threads>", "Number of threads to use when running queries. " +
+    .option("--threads <threads>", "The number of threads that can be used by CodeQL for database finalization and query execution. " +
-    "Default is to use all available cores.")
+    'By default, this command will use the same number of threads as previously set in the "init" command. ' +
+    'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
+    "hardware threads available in the system.")
     .option("--temp-dir <dir>", 'Directory to use for temporary files. Default is "./codeql-runner".')
     .option("--category <category>", "String used by Code Scanning for matching the analyses.")
     .option("--debug", "Print more verbose output", false)
@@ -262,8 +283,15 @@ program
             url: (0, util_1.parseGitHubUrl)(cmd.githubUrl),
         };
         const outputDir = cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-        const threads = (0, util_1.getThreadsFlag)(cmd.threads, logger);
+        let initEnv = {};
-        const memory = (0, util_1.getMemoryFlag)(cmd.ram);
+        try {
+            initEnv = loadTracerEnvironment(config);
+        }
+        catch (err) {
+            // The init command did not generate a tracer environment file
+        }
+        const threads = (0, util_1.getThreadsFlag)(cmd.threads || initEnv["CODEQL_THREADS"], logger);
+        const memory = (0, util_1.getMemoryFlag)(cmd.ram || initEnv["CODEQL_RAM"]);
         await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
         await (0, analyze_1.runQueries)(outputDir, memory, (0, util_1.getAddSnippetsFlag)(cmd.addSnippets), threads, cmd.category, config, logger);
         if (!cmd.upload) {

lib/util.js

@@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.GITHUB_DOTCOM_URL = void 0;
+exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.GITHUB_DOTCOM_URL = void 0;
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
 const path = __importStar(require("path"));
@@ -93,13 +93,13 @@ function getSystemReservedMemoryMegaBytes() {
     return 1024 * (process.platform === "win32" ? 1.5 : 1);
 }
 /**
- * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
+ * Get the value of the codeql `--ram` flag as configured by the `ram` input.
- * specified, the total available memory will be used minus a threshold
+ * If no value was specified, the total available memory will be used minus a
- * reserved for the OS.
+ * threshold reserved for the OS.
  *
- * @returns string
+ * @returns {number} the amount of RAM to use, in megabytes
  */
-function getMemoryFlag(userInput) {
+function getMemoryFlagValue(userInput) {
     let memoryToUseMegaBytes;
     if (userInput) {
         memoryToUseMegaBytes = Number(userInput);
@@ -113,7 +113,18 @@ function getMemoryFlag(userInput) {
         const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
         memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
     }
-    return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
+    return Math.floor(memoryToUseMegaBytes);
+}
+exports.getMemoryFlagValue = getMemoryFlagValue;
+/**
+ * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
+ * specified, the total available memory will be used minus a threshold
+ * reserved for the OS.
+ *
+ * @returns string
+ */
+function getMemoryFlag(userInput) {
+    return `--ram=${getMemoryFlagValue(userInput)}`;
 }
 exports.getMemoryFlag = getMemoryFlag;
 /**
@@ -130,14 +141,14 @@ function getAddSnippetsFlag(userInput) {
 }
 exports.getAddSnippetsFlag = getAddSnippetsFlag;
 /**
- * Get the codeql `--threads` value specified for the `threads` input.
+ * Get the value of the codeql `--threads` flag specified for the `threads`
- * If no value was specified, all available threads will be used.
+ * input. If no value was specified, all available threads will be used.
  *
  * The value will be capped to the number of available CPUs.
  *
- * @returns string
+ * @returns {number}
  */
-function getThreadsFlag(userInput, logger) {
+function getThreadsFlagValue(userInput, logger) {
     let numThreads;
     const maxThreads = os.cpus().length;
     if (userInput) {
@@ -159,7 +170,19 @@ function getThreadsFlag(userInput, logger) {
         // Default to using all threads
         numThreads = maxThreads;
     }
-    return `--threads=${numThreads}`;
+    return numThreads;
+}
+exports.getThreadsFlagValue = getThreadsFlagValue;
+/**
+ * Get the codeql `--threads` flag specified for the `threads` input.
+ * If no value was specified, all available threads will be used.
+ *
+ * The value will be capped to the number of available CPUs.
+ *
+ * @returns string
+ */
+function getThreadsFlag(userInput, logger) {
+    return `--threads=${getThreadsFlagValue(userInput, logger)}`;
 }
 exports.getThreadsFlag = getThreadsFlag;
 /**

pr-checks/checks/extractor-ram-threads.yml

@@ -0,0 +1,29 @@
+name: "Extractor ram and threads options test"
+description: "Tests passing RAM and threads limits to extractors"
+versions: ["latest"]
+os: ["ubuntu-latest"]
+steps:
+  - uses: ./../action/init
+    with:
+      languages: java
+      ram: 230
+      threads: 1
+  - name: Assert Results
+    shell: bash
+    run: |
+      if [ "${CODEQL_RAM}" != "230" ]; then
+        echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
+        exit 1
+      fi
+      if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
+        echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
+        exit 1
+      fi
+      if [ "${CODEQL_THREADS}" != "1" ]; then
+        echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
+        exit 1
+      fi
+      if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
+        echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
+        exit 1
+      fi

queries/inconsistent-action-input.ql

@@ -47,5 +47,7 @@ predicate areNotEquivalent(YAMLValue x, YAMLValue y) {
 from ActionDeclaration actionA, ActionDeclaration actionB, string inputName
 where actionA.getName() < actionB.getName() // prevent duplicates which are permutations of the names
   and areNotEquivalent(actionA.getInput(inputName), actionB.getInput(inputName))
+  // ram and threads inputs in different actions are supposed to have different description
+  and inputName != "ram" and inputName != "threads"
 select actionA, "Action $@ and action $@ both declare input $@, however their definitions are not identical. This may be confusing to users.",
   actionA, actionA.getName(), actionB, actionB.getName(), inputName, inputName

src/analyze-action-env.test.ts

@@ -0,0 +1,58 @@
+import test from "ava";
+import * as sinon from "sinon";
+
+import * as actionsUtil from "./actions-util";
+import * as analyze from "./analyze";
+import * as configUtils from "./config-utils";
+import { setupTests, setupActionsVars } from "./testing-utils";
+import * as util from "./util";
+
+setupTests(test);
+
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+
+test("analyze action with RAM & threads from environment variables", async (t) => {
+  await util.withTmpDir(async (tmpDir) => {
+    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    sinon
+      .stub(actionsUtil, "createStatusReportBase")
+      .resolves({} as actionsUtil.StatusReportBase);
+    sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    sinon.stub(configUtils, "getConfig").resolves({
+      languages: [],
+    } as unknown as configUtils.Config);
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("token").returns("fake-token");
+    requiredInputStub.withArgs("upload-database").returns("false");
+    const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+    optionalInputStub.withArgs("cleanup-level").returns("none");
+    setupActionsVars(tmpDir, tmpDir);
+
+    // When there are no action inputs for RAM and threads, the action uses
+    // environment variables (passed down from the init action) to set RAM and
+    // threads usage.
+    process.env["CODEQL_THREADS"] = "-1";
+    process.env["CODEQL_RAM"] = "4992";
+
+    const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+    const runQueriesStub = sinon.stub(analyze, "runQueries");
+    const analyzeAction = require("./analyze-action");
+
+    // When analyze-action.ts loads, it runs an async function from the top
+    // level but does not wait for it to finish. To ensure that calls to
+    // runFinalize and runQueries are correctly captured by spies, we explicitly
+    // wait for the action promise to complete before starting verification.
+    await analyzeAction.runPromise;
+
+    t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+    t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
+    t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+    t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
+  });
+});

src/analyze-action-input.test.ts

@@ -0,0 +1,59 @@
+import test from "ava";
+import * as sinon from "sinon";
+
+import * as actionsUtil from "./actions-util";
+import * as analyze from "./analyze";
+import * as configUtils from "./config-utils";
+import { setupTests, setupActionsVars } from "./testing-utils";
+import * as util from "./util";
+
+setupTests(test);
+
+// This test needs to be in its own file so that ava would run it in its own
+// nodejs process. The code being tested is in analyze-action.ts, which runs
+// immediately on load. So the file needs to be loaded during part of the test,
+// and that can happen only once per nodejs process. If multiple such tests are
+// in the same test file, ava would run them in the same nodejs process, and all
+// but the first test would fail.
+
+test("analyze action with RAM & threads from action inputs", async (t) => {
+  await util.withTmpDir(async (tmpDir) => {
+    process.env["GITHUB_SERVER_URL"] = "fake-server-url";
+    process.env["GITHUB_REPOSITORY"] = "fake/repository";
+    sinon
+      .stub(actionsUtil, "createStatusReportBase")
+      .resolves({} as actionsUtil.StatusReportBase);
+    sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
+    sinon.stub(configUtils, "getConfig").resolves({
+      languages: [],
+    } as unknown as configUtils.Config);
+    const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
+    requiredInputStub.withArgs("token").returns("fake-token");
+    requiredInputStub.withArgs("upload-database").returns("false");
+    const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
+    optionalInputStub.withArgs("cleanup-level").returns("none");
+    setupActionsVars(tmpDir, tmpDir);
+
+    process.env["CODEQL_THREADS"] = "1";
+    process.env["CODEQL_RAM"] = "4992";
+
+    // Action inputs have precedence over environment variables.
+    optionalInputStub.withArgs("threads").returns("-1");
+    optionalInputStub.withArgs("ram").returns("3012");
+
+    const runFinalizeStub = sinon.stub(analyze, "runFinalize");
+    const runQueriesStub = sinon.stub(analyze, "runQueries");
+    const analyzeAction = require("./analyze-action");
+
+    // When analyze-action.ts loads, it runs an async function from the top
+    // level but does not wait for it to finish. To ensure that calls to
+    // runFinalize and runQueries are correctly captured by spies, we explicitly
+    // wait for the action promise to complete before starting verification.
+    await analyzeAction.runPromise;
+
+    t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
+    t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
+    t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
+    t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
+  });
+});

src/analyze-action.ts

@@ -31,7 +31,7 @@ interface FinishStatusReport
   extends actionsUtil.StatusReportBase,
     AnalysisStatusReport {}
 
-async function sendStatusReport(
+export async function sendStatusReport(
   startedAt: Date,
   stats: AnalysisStatusReport | undefined,
   error?: Error
@@ -91,10 +91,12 @@ async function run() {
     };
     const outputDir = actionsUtil.getRequiredInput("output");
     const threads = util.getThreadsFlag(
-      actionsUtil.getOptionalInput("threads"),
+      actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
       logger
     );
-    const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram"));
+    const memory = util.getMemoryFlag(
+      actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
+    );
     await runFinalize(outputDir, threads, memory, config, logger);
     if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
       runStats = await runQueries(
@@ -188,9 +190,11 @@ async function run() {
   }
 }
 
+export const runPromise = run();
+
 async function runWrapper() {
   try {
-    await run();
+    await runPromise;
   } catch (error) {
     core.setFailed(`analyze action failed: ${error}`);
     console.log(error);

src/init-action.ts

@@ -32,6 +32,8 @@ import {
   getGitHubVersion,
   codeQlVersionAbove,
   enrichEnvironment,
+  getMemoryFlagValue,
+  getThreadsFlagValue,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -204,9 +206,20 @@ async function run() {
       );
     }
 
-    // Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
+    // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
-    const codeqlRam = process.env["CODEQL_RAM"] || "6500";
+    // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
-    core.exportVariable("CODEQL_RAM", codeqlRam);
+    // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+    // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+    // for details.
+    core.exportVariable(
+      "CODEQL_RAM",
+      process.env["CODEQL_RAM"] ||
+        getMemoryFlagValue(getOptionalInput("ram")).toString()
+    );
+    core.exportVariable(
+      "CODEQL_THREADS",
+      getThreadsFlagValue(getOptionalInput("threads"), logger).toString()
+    );
 
     const sourceRoot = path.resolve(
       getRequiredEnvParam("GITHUB_WORKSPACE"),

src/runner.ts

@@ -25,6 +25,8 @@ import {
   Mode,
   codeQlVersionAbove,
   enrichEnvironment,
+  getMemoryFlagValue,
+  getThreadsFlagValue,
 } from "./util";
 
 // eslint-disable-next-line import/no-commonjs
@@ -53,11 +55,15 @@ function getToolsDir(userInput: string | undefined): string {
 
 const codeqlEnvJsonFilename = "codeql-env.json";
 
+function loadTracerEnvironment(config: Config): { [name: string]: string } {
+  const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
+  return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
+}
+
 // Imports the environment from codeqlEnvJsonFilename if not already present
 function importTracerEnvironment(config: Config) {
   if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
-    const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
+    const env = loadTracerEnvironment(config);
-    const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
     for (const key of Object.keys(env)) {
       process.env[key] = env[key];
     }
@@ -109,6 +115,8 @@ interface InitArgs {
   githubAuth: string;
   githubAuthStdin: boolean;
   debug: boolean;
+  ram: string | undefined;
+  threads: string | undefined;
 }
 
 program
@@ -167,6 +175,18 @@ program
     "--trace-process-level <number>",
     "(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up."
   )
+  .option(
+    "--ram <number>",
+    "The amount of memory in MB that can be used by CodeQL extractors. " +
+      "By default, CodeQL extractors will use most of the memory available in the system. " +
+      'This input also sets the amount of memory that can later be used by the "analyze" command.'
+  )
+  .option(
+    "--threads <number>",
+    "The number of threads that can be used by CodeQL extractors. " +
+      "By default, CodeQL extractors will use all the hardware threads available in the system. " +
+      'This input also sets the number of threads that can later be used by the "analyze" command.'
+  )
   .action(async (cmd: InitArgs) => {
     const logger = getRunnerLogger(cmd.debug);
 
@@ -195,6 +215,17 @@ program
       const gitHubVersion = await getGitHubVersion(apiDetails);
       checkGitHubVersionInRange(gitHubVersion, logger, Mode.runner);
 
+      // Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
+      // CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
+      // threads it would ask extractors to use. See help text for the "--ram" and "--threads"
+      // options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
+      // for details.
+      process.env["CODEQL_RAM"] = getMemoryFlagValue(cmd.ram).toString();
+      process.env["CODEQL_THREADS"] = getThreadsFlagValue(
+        cmd.threads,
+        logger
+      ).toString();
+
       let codeql: CodeQL;
       if (cmd.codeqlPath !== undefined) {
         codeql = await getCodeQL(cmd.codeqlPath);
@@ -402,7 +433,10 @@ program
   )
   .option(
     "--ram <ram>",
-    "Amount of memory to use when running queries. Default is to use all available memory."
+    "The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
+      'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
+      'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
+      "memory available in the system."
   )
   .option(
     "--no-add-snippets",
@@ -410,8 +444,10 @@ program
   )
   .option(
     "--threads <threads>",
-    "Number of threads to use when running queries. " +
+    "The number of threads that can be used by CodeQL for database finalization and query execution. " +
-      "Default is to use all available cores."
+      'By default, this command will use the same number of threads as previously set in the "init" command. ' +
+      'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
+      "hardware threads available in the system."
   )
   .option(
     "--temp-dir <dir>",
@@ -447,8 +483,17 @@ program
 
       const outputDir =
         cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
-      const threads = getThreadsFlag(cmd.threads, logger);
+      let initEnv: { [name: string]: string } = {};
-      const memory = getMemoryFlag(cmd.ram);
+      try {
+        initEnv = loadTracerEnvironment(config);
+      } catch (err) {
+        // The init command did not generate a tracer environment file
+      }
+      const threads = getThreadsFlag(
+        cmd.threads || initEnv["CODEQL_THREADS"],
+        logger
+      );
+      const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
       await runFinalize(outputDir, threads, memory, config, logger);
       await runQueries(
         outputDir,

src/util.ts

@@ -85,13 +85,13 @@ function getSystemReservedMemoryMegaBytes(): number {
 }
 
 /**
- * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
+ * Get the value of the codeql `--ram` flag as configured by the `ram` input.
- * specified, the total available memory will be used minus a threshold
+ * If no value was specified, the total available memory will be used minus a
- * reserved for the OS.
+ * threshold reserved for the OS.
  *
- * @returns string
+ * @returns {number} the amount of RAM to use, in megabytes
  */
-export function getMemoryFlag(userInput: string | undefined): string {
+export function getMemoryFlagValue(userInput: string | undefined): number {
   let memoryToUseMegaBytes: number;
   if (userInput) {
     memoryToUseMegaBytes = Number(userInput);
@@ -104,7 +104,18 @@ export function getMemoryFlag(userInput: string | undefined): string {
     const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
     memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
   }
-  return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
+  return Math.floor(memoryToUseMegaBytes);
+}
+
+/**
+ * Get the codeql `--ram` flag as configured by the `ram` input. If no value was
+ * specified, the total available memory will be used minus a threshold
+ * reserved for the OS.
+ *
+ * @returns string
+ */
+export function getMemoryFlag(userInput: string | undefined): string {
+  return `--ram=${getMemoryFlagValue(userInput)}`;
 }
 
 /**
@@ -123,17 +134,17 @@ export function getAddSnippetsFlag(
 }
 
 /**
- * Get the codeql `--threads` value specified for the `threads` input.
+ * Get the value of the codeql `--threads` flag specified for the `threads`
- * If no value was specified, all available threads will be used.
+ * input. If no value was specified, all available threads will be used.
  *
  * The value will be capped to the number of available CPUs.
  *
- * @returns string
+ * @returns {number}
  */
-export function getThreadsFlag(
+export function getThreadsFlagValue(
   userInput: string | undefined,
   logger: Logger
-): string {
+): number {
   let numThreads: number;
   const maxThreads = os.cpus().length;
   if (userInput) {
@@ -158,7 +169,22 @@ export function getThreadsFlag(
     // Default to using all threads
     numThreads = maxThreads;
   }
-  return `--threads=${numThreads}`;
+  return numThreads;
+}
+
+/**
+ * Get the codeql `--threads` flag specified for the `threads` input.
+ * If no value was specified, all available threads will be used.
+ *
+ * The value will be capped to the number of available CPUs.
+ *
+ * @returns string
+ */
+export function getThreadsFlag(
+  userInput: string | undefined,
+  logger: Logger
+): string {
+  return `--threads=${getThreadsFlagValue(userInput, logger)}`;
 }
 
 /**