Merge pull request #606 from edoardopirovano/local-bundle

Allow local instead of downloaded CodeQL

.github/workflows/pr-checks.yml

@@ -294,6 +294,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       versions: ${{ steps.compare.outputs.versions }}
+      nightly-url: ${{ steps.get-url.outputs.nightly-url }}
 
     steps:
     - uses: actions/checkout@v2
@@ -879,3 +880,23 @@ jobs:
         # Deliberately don't use TEST_MODE here. This is specifically testing
         # the compatibility with the API.
         runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
+
+  multi-language-repo_test-local-codeql:
+    needs: [check-js, check-node-modules, check-codeql-versions]
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      run: |
+        wget ${{ needs.check-codeql-versions.outputs.nightly-url }}
+        mkdir ../action
+        mv * .github ../action/
+        mv ../action/tests/multi-language-repo/{*,.github} .
+        mv ../action/.github/workflows .github
+    - uses: ./../action/init
+      with:
+        tools: ../action/codeql-bundle.tar.gz
+    - name: Build code
+      run: ./build.sh
+    - uses: ./../action/analyze

lib/codeql.js

@@ -154,50 +154,58 @@ async function setupCodeQL(codeqlURL, apiDetails, tempDir, toolCacheDir, variant
         if (forceLatest) {
             codeqlURL = undefined;
         }
-        const codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
+        let codeqlFolder;
-        const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+        let codeqlURLVersion;
-        // If we find the specified version, we always use that.
+        if (codeqlURL && !codeqlURL.startsWith("http")) {
-        let codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
+            codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
-        // If we don't find the requested version, in some cases we may allow a
+            codeqlURLVersion = "local";
-        // different version to save download time if the version hasn't been
-        // specified explicitly (in which case we always honor it).
-        if (!codeqlFolder && !codeqlURL && !forceLatest) {
-            const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
-            if (codeqlVersions.length === 1) {
-                const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
-                if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-                    logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
-                    codeqlFolder = tmpCodeqlFolder;
-                }
-            }
-        }
-        if (codeqlFolder) {
-            logger.debug(`CodeQL found in cache ${codeqlFolder}`);
         }
         else {
-            if (!codeqlURL) {
+            codeqlURLVersion = getCodeQLURLVersion(codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`);
-                codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+            const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+            // If we find the specified version, we always use that.
+            codeqlFolder = toolcache.find("CodeQL", codeqlURLSemVer, toolCacheDir, logger);
+            // If we don't find the requested version, in some cases we may allow a
+            // different version to save download time if the version hasn't been
+            // specified explicitly (in which case we always honor it).
+            if (!codeqlFolder && !codeqlURL && !forceLatest) {
+                const codeqlVersions = toolcache.findAllVersions("CodeQL", toolCacheDir, logger);
+                if (codeqlVersions.length === 1) {
+                    const tmpCodeqlFolder = toolcache.find("CodeQL", codeqlVersions[0], toolCacheDir, logger);
+                    if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+                        logger.debug(`CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`);
+                        codeqlFolder = tmpCodeqlFolder;
+                    }
+                }
             }
-            const parsedCodeQLURL = new URL(codeqlURL);
+            if (codeqlFolder) {
-            const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
+                logger.debug(`CodeQL found in cache ${codeqlFolder}`);
-            const headers = { accept: "application/octet-stream" };
-            // We only want to provide an authorization header if we are downloading
-            // from the same GitHub instance the Action is running on.
-            // This avoids leaking Enterprise tokens to dotcom.
-            // We also don't want to send an authorization header if there's already a token provided in the URL.
-            if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
-                parsedQueryString["token"] === undefined) {
-                logger.debug("Downloading CodeQL bundle with token.");
-                headers.authorization = `token ${apiDetails.auth}`;
             }
             else {
-                logger.debug("Downloading CodeQL bundle without token.");
+                if (!codeqlURL) {
+                    codeqlURL = await getCodeQLBundleDownloadURL(apiDetails, variant, logger);
+                }
+                const parsedCodeQLURL = new URL(codeqlURL);
+                const parsedQueryString = query_string_1.default.parse(parsedCodeQLURL.search);
+                const headers = { accept: "application/octet-stream" };
+                // We only want to provide an authorization header if we are downloading
+                // from the same GitHub instance the Action is running on.
+                // This avoids leaking Enterprise tokens to dotcom.
+                // We also don't want to send an authorization header if there's already a token provided in the URL.
+                if (codeqlURL.startsWith(`${apiDetails.url}/`) &&
+                    parsedQueryString["token"] === undefined) {
+                    logger.debug("Downloading CodeQL bundle with token.");
+                    headers.authorization = `token ${apiDetails.auth}`;
+                }
+                else {
+                    logger.debug("Downloading CodeQL bundle without token.");
+                }
+                logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
+                const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
+                logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+                const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
+                codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
             }
-            logger.info(`Downloading CodeQL tools from ${codeqlURL}. This may take a while.`);
-            const codeqlPath = await toolcache.downloadTool(codeqlURL, tempDir, headers);
-            logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-            const codeqlExtracted = await toolcache.extractTar(codeqlPath, tempDir, logger);
-            codeqlFolder = await toolcache.cacheDir(codeqlExtracted, "CodeQL", codeqlURLSemVer, toolCacheDir, logger);
         }
         let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
         if (process.platform === "win32") {

src/codeql.ts

@@ -318,96 +318,101 @@ export async function setupCodeQL(
     if (forceLatest) {
       codeqlURL = undefined;
     }
-
+    let codeqlFolder: string;
-    const codeqlURLVersion = getCodeQLURLVersion(
+    let codeqlURLVersion: string;
-      codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
+    if (codeqlURL && !codeqlURL.startsWith("http")) {
-    );
+      codeqlFolder = await toolcache.extractTar(codeqlURL, tempDir, logger);
-    const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
+      codeqlURLVersion = "local";
-
-    // If we find the specified version, we always use that.
-    let codeqlFolder = toolcache.find(
-      "CodeQL",
-      codeqlURLSemVer,
-      toolCacheDir,
-      logger
-    );
-
-    // If we don't find the requested version, in some cases we may allow a
-    // different version to save download time if the version hasn't been
-    // specified explicitly (in which case we always honor it).
-    if (!codeqlFolder && !codeqlURL && !forceLatest) {
-      const codeqlVersions = toolcache.findAllVersions(
-        "CodeQL",
-        toolCacheDir,
-        logger
-      );
-      if (codeqlVersions.length === 1) {
-        const tmpCodeqlFolder = toolcache.find(
-          "CodeQL",
-          codeqlVersions[0],
-          toolCacheDir,
-          logger
-        );
-        if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
-          logger.debug(
-            `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
-          );
-          codeqlFolder = tmpCodeqlFolder;
-        }
-      }
-    }
-
-    if (codeqlFolder) {
-      logger.debug(`CodeQL found in cache ${codeqlFolder}`);
     } else {
-      if (!codeqlURL) {
+      codeqlURLVersion = getCodeQLURLVersion(
-        codeqlURL = await getCodeQLBundleDownloadURL(
+        codeqlURL || `/${CODEQL_BUNDLE_VERSION}/`
-          apiDetails,
+      );
-          variant,
+      const codeqlURLSemVer = convertToSemVer(codeqlURLVersion, logger);
-          logger
-        );
-      }
 
-      const parsedCodeQLURL = new URL(codeqlURL);
+      // If we find the specified version, we always use that.
-      const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
+      codeqlFolder = toolcache.find(
-      const headers: IHeaders = { accept: "application/octet-stream" };
-      // We only want to provide an authorization header if we are downloading
-      // from the same GitHub instance the Action is running on.
-      // This avoids leaking Enterprise tokens to dotcom.
-      // We also don't want to send an authorization header if there's already a token provided in the URL.
-      if (
-        codeqlURL.startsWith(`${apiDetails.url}/`) &&
-        parsedQueryString["token"] === undefined
-      ) {
-        logger.debug("Downloading CodeQL bundle with token.");
-        headers.authorization = `token ${apiDetails.auth}`;
-      } else {
-        logger.debug("Downloading CodeQL bundle without token.");
-      }
-      logger.info(
-        `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
-      );
-      const codeqlPath = await toolcache.downloadTool(
-        codeqlURL,
-        tempDir,
-        headers
-      );
-      logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
-
-      const codeqlExtracted = await toolcache.extractTar(
-        codeqlPath,
-        tempDir,
-        logger
-      );
-      codeqlFolder = await toolcache.cacheDir(
-        codeqlExtracted,
         "CodeQL",
         codeqlURLSemVer,
         toolCacheDir,
         logger
       );
-    }
 
+      // If we don't find the requested version, in some cases we may allow a
+      // different version to save download time if the version hasn't been
+      // specified explicitly (in which case we always honor it).
+      if (!codeqlFolder && !codeqlURL && !forceLatest) {
+        const codeqlVersions = toolcache.findAllVersions(
+          "CodeQL",
+          toolCacheDir,
+          logger
+        );
+        if (codeqlVersions.length === 1) {
+          const tmpCodeqlFolder = toolcache.find(
+            "CodeQL",
+            codeqlVersions[0],
+            toolCacheDir,
+            logger
+          );
+          if (fs.existsSync(path.join(tmpCodeqlFolder, "pinned-version"))) {
+            logger.debug(
+              `CodeQL in cache overriding the default ${CODEQL_BUNDLE_VERSION}`
+            );
+            codeqlFolder = tmpCodeqlFolder;
+          }
+        }
+      }
+
+      if (codeqlFolder) {
+        logger.debug(`CodeQL found in cache ${codeqlFolder}`);
+      } else {
+        if (!codeqlURL) {
+          codeqlURL = await getCodeQLBundleDownloadURL(
+            apiDetails,
+            variant,
+            logger
+          );
+        }
+
+        const parsedCodeQLURL = new URL(codeqlURL);
+        const parsedQueryString = queryString.parse(parsedCodeQLURL.search);
+        const headers: IHeaders = { accept: "application/octet-stream" };
+        // We only want to provide an authorization header if we are downloading
+        // from the same GitHub instance the Action is running on.
+        // This avoids leaking Enterprise tokens to dotcom.
+        // We also don't want to send an authorization header if there's already a token provided in the URL.
+        if (
+          codeqlURL.startsWith(`${apiDetails.url}/`) &&
+          parsedQueryString["token"] === undefined
+        ) {
+          logger.debug("Downloading CodeQL bundle with token.");
+          headers.authorization = `token ${apiDetails.auth}`;
+        } else {
+          logger.debug("Downloading CodeQL bundle without token.");
+        }
+        logger.info(
+          `Downloading CodeQL tools from ${codeqlURL}. This may take a while.`
+        );
+        const codeqlPath = await toolcache.downloadTool(
+          codeqlURL,
+          tempDir,
+          headers
+        );
+        logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
+
+        const codeqlExtracted = await toolcache.extractTar(
+          codeqlPath,
+          tempDir,
+          logger
+        );
+        codeqlFolder = await toolcache.cacheDir(
+          codeqlExtracted,
+          "CodeQL",
+          codeqlURLSemVer,
+          toolCacheDir,
+          logger
+        );
+      }
+    }
     let codeqlCmd = path.join(codeqlFolder, "codeql", "codeql");
     if (process.platform === "win32") {
       codeqlCmd += ".exe";