Use CLI's own baseline LOC counting

.github/workflows/__go-custom-queries.yml

@@ -58,3 +58,5 @@ jobs:
     - uses: ./../action/analyze
       env:
         TEST_MODE: true
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__go-custom-tracing-autobuild.yml

@@ -61,3 +61,4 @@ jobs:
         fi
     env:
       CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'true'
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__go-custom-tracing.yml

@@ -59,3 +59,4 @@ jobs:
         TEST_MODE: true
     env:
       CODEQL_EXTRACTOR_GO_BUILD_TRACING: 'true'
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__javascript-source-root.yml

@@ -57,3 +57,5 @@ jobs:
           echo "Did not find a JavaScript database"
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__multi-language-autodetect.yml

@@ -84,3 +84,5 @@ jobs:
           echo "Did not create a database for Python, or created it in the wrong location."
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__packaging-config-inputs-js.yml

@@ -63,3 +63,5 @@ jobs:
           echo "Did not match expected rules '$EXPECTED_RULES'."
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__packaging-config-js.yml

@@ -62,3 +62,5 @@ jobs:
           echo "Did not match expected rules '$EXPECTED_RULES'."
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__packaging-inputs-js.yml

@@ -63,3 +63,5 @@ jobs:
           echo "Did not match expected rules '$EXPECTED_RULES'."
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__remote-config.yml

@@ -56,3 +56,5 @@ jobs:
     - uses: ./../action/analyze
       env:
         TEST_MODE: true
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__rubocop-multi-language.yml

@@ -63,3 +63,5 @@ jobs:
         sarif_file: rubocop.sarif
       env:
         TEST_MODE: true
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__split-workflow.yml

@@ -76,3 +76,5 @@ jobs:
           echo "Did not match expected rules '$EXPECTED_RULES'."
           exit 1
         fi
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__test-local-codeql.yml

@@ -50,3 +50,5 @@ jobs:
     - uses: ./../action/analyze
       env:
         TEST_MODE: true
+    env:
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

.github/workflows/__test-proxy.yml

@@ -44,6 +44,7 @@ jobs:
         TEST_MODE: true
     env:
       https_proxy: http://squid-proxy:3128
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true
     container:
       image: ubuntu:18.04
       options: --dns 127.0.0.1

.github/workflows/__test-ruby.yml

@@ -53,3 +53,4 @@ jobs:
         fi
     env:
       CODEQL_ENABLE_EXPERIMENTAL_FEATURES: 'true'
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

lib/analyze.js

@@ -108,12 +108,19 @@ async function finalizeDatabaseCreation(config, threadsFlag, memoryFlag, logger)
 // Runs queries and creates sarif files in the given folder
 async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, automationDetailsId, config, logger) {
     const statusReport = {};
-    // count the number of lines in the background
-    const locPromise = (0, count_loc_1.countLoc)(path.resolve(), 
-    // config.paths specifies external directories. the current
-    // directory is included in the analysis by default. Replicate
-    // that here.
-    config.paths, config.pathsIgnore, config.languages, logger);
+    let locPromise = Promise.resolve({});
+    const cliCanCountBaseline = await cliCanCountLoC();
+    const debugMode = process.env["INTERNAL_CODEQL_ACTION_DEBUG_LOC"] ||
+        process.env["ACTIONS_RUNNER_DEBUG"] ||
+        process.env["ACTIONS_STEP_DEBUG"];
+    if (!cliCanCountBaseline || debugMode) {
+        // count the number of lines in the background
+        locPromise = (0, count_loc_1.countLoc)(path.resolve(), 
+        // config.paths specifies external directories. the current
+        // directory is included in the analysis by default. Replicate
+        // that here.
+        config.paths, config.pathsIgnore, config.languages, logger);
+    }
     for (const language of config.languages) {
         const queries = config.queries[language];
         const packsWithVersion = config.packs[language] || [];
@@ -166,12 +173,16 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
             const startTimeInterpretResults = new Date().getTime();
             const sarifFile = path.join(sarifFolder, `${language}.sarif`);
             const analysisSummary = await runInterpretResults(language, querySuitePaths, sarifFile);
-            await injectLinesOfCode(sarifFile, language, locPromise);
+            if (!cliCanCountBaseline)
+                await injectLinesOfCode(sarifFile, language, locPromise);
             statusReport[`interpret_results_${language}_duration_ms`] =
                 new Date().getTime() - startTimeInterpretResults;
             logger.endGroup();
             logger.info(analysisSummary);
-            printLinesOfCodeSummary(logger, language, await locPromise);
+            if (!cliCanCountBaseline || debugMode)
+                printLinesOfCodeSummary(logger, language, await locPromise);
+            if (cliCanCountBaseline)
+                logger.info(await runPrintLinesOfCode(language));
         }
         catch (e) {
             logger.info(String(e));
@@ -188,6 +199,14 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
         const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
         return await codeql.databaseInterpretResults(databasePath, queries, sarifFile, addSnippetsFlag, threadsFlag, automationDetailsId);
     }
+    async function cliCanCountLoC() {
+        return await util.codeQlVersionAbove(await (0, codeql_1.getCodeQL)(config.codeQLCmd), codeql_1.CODEQL_VERSION_COUNTS_LINES);
+    }
+    async function runPrintLinesOfCode(language) {
+        const databasePath = util.getCodeQLDatabasePath(config, language);
+        const codeql = await (0, codeql_1.getCodeQL)(config.codeQLCmd);
+        return await codeql.databasePrintBaseline(databasePath);
+    }
     async function runQueryGroup(language, type, querySuiteContents, searchPath) {
         const databasePath = util.getCodeQLDatabasePath(config, language);
         // Pass the queries to codeql using a file instead of using the command

lib/codeql.js

@@ -22,7 +22,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getExtraOptions = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_NEW_TRACING = exports.CommandInvocationError = void 0;
+exports.getExtraOptions = exports.getCachedCodeQL = exports.setCodeQL = exports.getCodeQL = exports.convertToSemVer = exports.getCodeQLURLVersion = exports.setupCodeQL = exports.getCodeQLActionRepository = exports.CODEQL_VERSION_COUNTS_LINES = exports.CODEQL_VERSION_NEW_TRACING = exports.CommandInvocationError = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const toolrunner = __importStar(require("@actions/exec/lib/toolrunner"));
@@ -72,6 +72,7 @@ const CODEQL_VERSION_METRICS = "2.5.5";
 const CODEQL_VERSION_GROUP_RULES = "2.5.5";
 const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
 exports.CODEQL_VERSION_NEW_TRACING = "2.6.0"; // Use multi-language (>= 2.5.6) and indirect (>= 2.6.0) tracing.
+exports.CODEQL_VERSION_COUNTS_LINES = "2.6.2";
 function getCodeQLBundleName() {
     let platform;
     if (process.platform === "win32") {
@@ -323,6 +324,7 @@ function setCodeQL(partialCodeql) {
         databaseBundle: resolveFunction(partialCodeql, "databaseBundle"),
         databaseRunQueries: resolveFunction(partialCodeql, "databaseRunQueries"),
         databaseInterpretResults: resolveFunction(partialCodeql, "databaseInterpretResults"),
+        databasePrintBaseline: resolveFunction(partialCodeql, "databasePrintBaseline"),
     };
     return cachedCodeQL;
 }
@@ -559,6 +561,15 @@ async function getCodeQLForCmd(cmd, checkVersion) {
             // capture stdout, which contains analysis summaries
             return await runTool(cmd, codeqlArgs);
         },
+        async databasePrintBaseline(databasePath) {
+            const codeqlArgs = [
+                "database",
+                "print-baseline",
+                ...getExtraOptionsFromEnv(["database", "print-baseline"]),
+                databasePath,
+            ];
+            return await runTool(cmd, codeqlArgs);
+        },
         /**
          * Download specified packs into the package cache. If the specified
          * package and version already exists (e.g., from a previous analysis run),

pr-checks/sync.py

@@ -79,6 +79,8 @@ for file in os.listdir('checks'):
         if key in checkSpecification:
             checkJob[key] = checkSpecification[key]
 
+    checkJob['env'] = checkJob.get('env', {})
+    checkJob['env']['INTERNAL_CODEQL_ACTION_DEBUG_LOC'] = True
     checkName = file[:len(file) - 4]
 
     with open(f"../.github/workflows/__{checkName}.yml", 'w') as output_stream:

src/analyze.ts

@@ -5,7 +5,7 @@ import * as toolrunner from "@actions/exec/lib/toolrunner";
 import * as yaml from "js-yaml";
 
 import * as analysisPaths from "./analysis-paths";
-import { getCodeQL } from "./codeql";
+import { CODEQL_VERSION_COUNTS_LINES, getCodeQL } from "./codeql";
 import * as configUtils from "./config-utils";
 import { countLoc } from "./count-loc";
 import { isScannedLanguage, Language } from "./languages";
@@ -194,17 +194,27 @@ export async function runQueries(
 ): Promise<QueriesStatusReport> {
   const statusReport: QueriesStatusReport = {};
 
-  // count the number of lines in the background
-  const locPromise = countLoc(
-    path.resolve(),
-    // config.paths specifies external directories. the current
-    // directory is included in the analysis by default. Replicate
-    // that here.
-    config.paths,
-    config.pathsIgnore,
-    config.languages,
-    logger
+  let locPromise: Promise<Partial<Record<Language, number>>> = Promise.resolve(
+    {}
   );
+  const cliCanCountBaseline = await cliCanCountLoC();
+  const debugMode =
+    process.env["INTERNAL_CODEQL_ACTION_DEBUG_LOC"] ||
+    process.env["ACTIONS_RUNNER_DEBUG"] ||
+    process.env["ACTIONS_STEP_DEBUG"];
+  if (!cliCanCountBaseline || debugMode) {
+    // count the number of lines in the background
+    locPromise = countLoc(
+      path.resolve(),
+      // config.paths specifies external directories. the current
+      // directory is included in the analysis by default. Replicate
+      // that here.
+      config.paths,
+      config.pathsIgnore,
+      config.languages,
+      logger
+    );
+  }
 
   for (const language of config.languages) {
     const queries = config.queries[language];
@@ -295,12 +305,15 @@ export async function runQueries(
         querySuitePaths,
         sarifFile
       );
-      await injectLinesOfCode(sarifFile, language, locPromise);
+      if (!cliCanCountBaseline)
+        await injectLinesOfCode(sarifFile, language, locPromise);
       statusReport[`interpret_results_${language}_duration_ms`] =
         new Date().getTime() - startTimeInterpretResults;
       logger.endGroup();
       logger.info(analysisSummary);
-      printLinesOfCodeSummary(logger, language, await locPromise);
+      if (!cliCanCountBaseline || debugMode)
+        printLinesOfCodeSummary(logger, language, await locPromise);
+      if (cliCanCountBaseline) logger.info(await runPrintLinesOfCode(language));
     } catch (e) {
       logger.info(String(e));
       if (e instanceof Error) {
@@ -333,6 +346,19 @@ export async function runQueries(
     );
   }
 
+  async function cliCanCountLoC() {
+    return await util.codeQlVersionAbove(
+      await getCodeQL(config.codeQLCmd),
+      CODEQL_VERSION_COUNTS_LINES
+    );
+  }
+
+  async function runPrintLinesOfCode(language: Language): Promise<string> {
+    const databasePath = util.getCodeQLDatabasePath(config, language);
+    const codeql = await getCodeQL(config.codeQLCmd);
+    return await codeql.databasePrintBaseline(databasePath);
+  }
+
   async function runQueryGroup(
     language: Language,
     type: string,
@@ -361,7 +387,6 @@ export async function runQueries(
     return querySuitePath;
   }
 }
-
 function createQuerySuiteContents(queries: string[]) {
   return queries.map((q: string) => `- query: ${q}`).join("\n");
 }

src/codeql.ts

@@ -148,6 +148,10 @@ export interface CodeQL {
     threadsFlag: string,
     automationDetailsId: string | undefined
   ): Promise<string>;
+  /**
+   * Run 'codeql database print-baseline'.
+   */
+  databasePrintBaseline(databasePath: string): Promise<string>;
 }
 
 export interface ResolveLanguagesOutput {
@@ -209,6 +213,7 @@ const CODEQL_VERSION_METRICS = "2.5.5";
 const CODEQL_VERSION_GROUP_RULES = "2.5.5";
 const CODEQL_VERSION_SARIF_GROUP = "2.5.3";
 export const CODEQL_VERSION_NEW_TRACING = "2.6.0"; // Use multi-language (>= 2.5.6) and indirect (>= 2.6.0) tracing.
+export const CODEQL_VERSION_COUNTS_LINES = "2.6.2";
 
 function getCodeQLBundleName(): string {
   let platform: string;
@@ -556,6 +561,10 @@ export function setCodeQL(partialCodeql: Partial<CodeQL>): CodeQL {
       partialCodeql,
       "databaseInterpretResults"
     ),
+    databasePrintBaseline: resolveFunction(
+      partialCodeql,
+      "databasePrintBaseline"
+    ),
   };
   return cachedCodeQL;
 }
@@ -860,6 +869,15 @@ async function getCodeQLForCmd(
       // capture stdout, which contains analysis summaries
       return await runTool(cmd, codeqlArgs);
     },
+    async databasePrintBaseline(databasePath: string): Promise<string> {
+      const codeqlArgs = [
+        "database",
+        "print-baseline",
+        ...getExtraOptionsFromEnv(["database", "print-baseline"]),
+        databasePath,
+      ];
+      return await runTool(cmd, codeqlArgs);
+    },
 
     /**
      * Download specified packs into the package cache. If the specified