Add extra integration test for packaging

Also, update the options and inputs documentation.

.github/workflows/pr-checks.yml

@@ -101,6 +101,7 @@ jobs:
         fi
 
   # Packaging test that runs against a javascript database
+  # Specifying packs in the config file.
   test-packaging-javascript-config:
     needs: [check-js, check-node-modules]
     runs-on: ubuntu-latest
@@ -143,7 +144,8 @@ jobs:
           exit 1
         fi
 
-  # tests that we can run packages through actions inputs
+  # Packaging test that runs against a javascript database
+  # Specifying packs as an input.
   test-packaging-javascript-inputs:
     needs: [check-js, check-node-modules]
     runs-on: ubuntu-latest
@@ -187,6 +189,52 @@ jobs:
           exit 1
         fi
 
+  # Packaging test that runs against a javascript database
+  # Specifying packs in the config file and inputs.
+  test-packaging-javascript-config-and-inputs:
+    needs: [check-js, check-node-modules]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        config-file: ".github/codeql/codeql-config-packaging3.yml"
+        packs: +dsp-testing/codeql-pack1@0.0.4
+        languages: javascript
+        # TODO: this can be removed when cli v2.5.6 is released and available in the tool cache
+        tools: https://github.com/dsp-testing/aeisenberg-codeql-action-packaging/releases/download/codeql-bundle-20210615/codeql-bundle-linux64.tar.gz
+
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      with:
+        output: "${{ runner.temp }}/results"
+      env:
+        TEST_MODE: true
+    - name: Assert Results
+      run: |
+        cd "$RUNNER_TEMP/results"
+        # We should have 3 hits from these rules
+        EXPECTED_RULES="javascript/example/empty-or-one-block javascript/example/empty-or-one-block javascript/example/two-block"
+
+        # use tr to replace newlines with spaces and xargs to trim leading and trailing whitespace
+        RULES="$(cat javascript.sarif | jq -r '.runs[0].results[].ruleId' | sort | tr "\n" " " | xargs)"
+        echo "Found matching rules '$RULES'"
+        if [ "$RULES" != "$EXPECTED_RULES" ]; then
+          echo "Did not match expected rules '$EXPECTED_RULES'."
+          exit 1
+        fi
+
+
   # Identify the CodeQL tool versions to integration test against.
   check-codeql-versions:
     needs: [check-js, check-node-modules]

init/action.yml

@@ -24,11 +24,12 @@ inputs:
     required: false
   packs:
     description: >-
-      Comma-separated list of packs to run. Reference a pack in the format `scope/name[@version]`. If `version` is not
+      [Experimental] Comma-separated list of packs to run. Reference a pack in the format `scope/name[@version]`. If `version` is not
       specified, then the latest version of the pack is used. By default, this overrides the same setting in a
       configuration file; prefix with "+" to use both sets of packs.
 
-      This input is only available in single-language analyses.
+      This input is only available in single-language analyses. To use packs in multi-language
+      analyses, you must specify packs in the codeql-config.yml file.
     required: false
   external-repository-token:
     description: A token for fetching external config files and queries if they reside in a private repository.

lib/config-utils.js

@@ -425,7 +425,7 @@ async function getDefaultConfig(languagesInput, queriesInput, packsInput, dbLoca
     if (queriesInput) {
         await addQueriesFromWorkflow(codeQL, queriesInput, languages, queries, tempDir, checkoutPath, apiDetails, logger);
     }
-    const packs = (_a = parsePacksInput(packsInput, languages), (_a !== null && _a !== void 0 ? _a : {}));
+    const packs = (_a = parsePacksFromInput(packsInput, languages), (_a !== null && _a !== void 0 ? _a : {}));
     return {
         languages,
         queries,
@@ -584,13 +584,13 @@ function parsePacksFromConfig(packsByLanguage, languages, configFile) {
     return packs;
 }
 exports.parsePacksFromConfig = parsePacksFromConfig;
-function parsePacksInput(packsInput, languages) {
+function parsePacksFromInput(packsInput, languages) {
     var _a;
     if (!((_a = packsInput) === null || _a === void 0 ? void 0 : _a.trim())) {
         return undefined;
     }
     if (languages.length > 1) {
-        throw new Error("Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by library.");
+        throw new Error("Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language.");
     }
     else if (languages.length === 0) {
         throw new Error("No languages specified. Cannot process the packs input.");
@@ -599,7 +599,7 @@ function parsePacksInput(packsInput, languages) {
     if (packsInput.startsWith("+")) {
         packsInput = packsInput.substring(1).trim();
         if (!packsInput) {
-            throw new Error("Remove the '+' from the packs input.");
+            throw new Error("A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs.");
         }
     }
     return {
@@ -632,7 +632,7 @@ function toPackWithVersion(packStr, configFile) {
 }
 // exported for testing
 function parsePacks(rawPacksFromConfig, rawPacksInput, languages, configFile) {
-    const packsFromInput = parsePacksInput(rawPacksInput, languages);
+    const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
     const packsFomConfig = parsePacksFromConfig(rawPacksFromConfig, languages, configFile);
     if (!packsFromInput) {
         return packsFomConfig;

lib/config-utils.test.js

@@ -852,7 +852,7 @@ ava_1.default("input and config", parseInputAndConfigMacro, ["a/b", "c/d"], " +e
 });
 ava_1.default("input with no language", parseInputAndConfigErrorMacro, {}, "c/d", [], /No languages specified/);
 ava_1.default("input with two languages", parseInputAndConfigErrorMacro, {}, "c/d", [languages_1.Language.cpp, languages_1.Language.csharp], /multi-language analysis/);
-ava_1.default("input with + only", parseInputAndConfigErrorMacro, {}, " + ", [languages_1.Language.cpp], /Remove the '\+'/);
+ava_1.default("input with + only", parseInputAndConfigErrorMacro, {}, " + ", [languages_1.Language.cpp], /remove the '\+'/);
 ava_1.default("input with invalid pack name", parseInputAndConfigErrorMacro, {}, " xxx", [languages_1.Language.cpp], /"xxx" is not a valid pack/);
 // errors
 // input w invalid pack name

lib/runner.js

@@ -90,11 +90,12 @@ program
     .option("--github-auth-stdin", "Read GitHub Apps token or personal access token from stdin.")
     .option("--languages <languages>", "Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.")
     .option("--queries <queries>", "Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.")
-    .option("--packs <packs>", `Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
+    .option("--packs <packs>", `[Experimental] Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
     specified, then the latest version of the pack is used. By default, this overrides the same setting in a
     configuration file; prefix with "+" to use both sets of packs.
 
-    This option is only available in single-language analyses.`)
+    This option is only available in single-language analyses. To use packs in multi-language
+    analyses, you must specify packs in the codeql-config.yml file.`)
     .option("--config-file <file>", "Path to config file.")
     .option("--codeql-path <path>", "Path to a copy of the CodeQL CLI executable to use. Otherwise downloads a copy.")
     .option("--temp-dir <dir>", 'Directory to use for temporary files. Default is "./codeql-runner".')

src/config-utils.test.ts

@@ -1596,7 +1596,7 @@ test(
   {},
   " + ",
   [Language.cpp],
-  /Remove the '\+'/
+  /remove the '\+'/
 );
 
 test(

src/config-utils.ts

@@ -843,7 +843,7 @@ export async function getDefaultConfig(
     );
   }
 
-  const packs = parsePacksInput(packsInput, languages) ?? {};
+  const packs = parsePacksFromInput(packsInput, languages) ?? {};
 
   return {
     languages,
@@ -1075,7 +1075,7 @@ export function parsePacksFromConfig(
   return packs;
 }
 
-function parsePacksInput(
+function parsePacksFromInput(
   packsInput: string | undefined,
   languages: Language[]
 ): Packs | undefined {
@@ -1085,7 +1085,7 @@ function parsePacksInput(
 
   if (languages.length > 1) {
     throw new Error(
-      "Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by library."
+      "Cannot specify a 'packs' input in a multi-language analysis. Use a codeql-config.yml file instead and specify packs by language."
     );
   } else if (languages.length === 0) {
     throw new Error("No languages specified. Cannot process the packs input.");
@@ -1095,7 +1095,9 @@ function parsePacksInput(
   if (packsInput.startsWith("+")) {
     packsInput = packsInput.substring(1).trim();
     if (!packsInput) {
-      throw new Error("Remove the '+' from the packs input.");
+      throw new Error(
+        "A '+' was used in the 'packs' input to specify that you wished to add some packs to your CodeQL analysis. However, no packs were specified. Please either remove the '+' or specify some packs."
+      );
     }
   }
 
@@ -1139,7 +1141,7 @@ export function parsePacks(
   languages: Language[],
   configFile: string
 ) {
-  const packsFromInput = parsePacksInput(rawPacksInput, languages);
+  const packsFromInput = parsePacksFromInput(rawPacksInput, languages);
   const packsFomConfig = parsePacksFromConfig(
     rawPacksFromConfig,
     languages,

src/runner.ts

@@ -132,11 +132,12 @@ program
   )
   .option(
     "--packs <packs>",
-    `Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
+    `[Experimental] Comma-separated list of packs to run. Reference a pack in the format scope/name[@version]. If version is not
     specified, then the latest version of the pack is used. By default, this overrides the same setting in a
     configuration file; prefix with "+" to use both sets of packs.
 
-    This option is only available in single-language analyses.`
+    This option is only available in single-language analyses. To use packs in multi-language
+    analyses, you must specify packs in the codeql-config.yml file.`
   )
   .option("--config-file <file>", "Path to config file.")
   .option(

tests/multi-language-repo/.github/codeql/codeql-config-packaging3.yml

@@ -0,0 +1,9 @@
+name: Pack testing in the CodeQL Action
+
+disable-default-queries: true
+packs:
+  javascript:
+    - dsp-testing/codeql-pack2 # latest
+paths-ignore:
+  - tests
+  - lib