Merge pull request #179 from github/split_builtin_custom_queries

Split up builtin and custom queries

lib/analyze.js

@@ -39,35 +39,48 @@ async function finalizeDatabaseCreation(config, logger) {
 }
 // Runs queries and creates sarif files in the given folder
 async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag, config, logger) {
-    const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+    const statusReport = {};
     for (const language of config.languages) {
         logger.startGroup(`Analyzing ${language}`);
-        const queries = config.queries[language] || [];
+        const queries = config.queries[language];
-        if (queries.length === 0) {
+        if (queries.builtin.length === 0 && queries.custom.length === 0) {
             throw new Error(`Unable to analyse ${language} as no queries were selected for this language`);
         }
         try {
-            const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
+            for (const type of ["builtin", "custom"]) {
-            // Pass the queries to codeql using a file instead of using the command
+                if (queries[type].length > 0) {
-            // line to avoid command line length restrictions, particularly on windows.
+                    const startTime = new Date().getTime();
-            const querySuite = `${databasePath}-queries.qls`;
+                    const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
-            const querySuiteContents = queries.map((q) => `- query: ${q}`).join("\n");
+                    // Pass the queries to codeql using a file instead of using the command
-            fs.writeFileSync(querySuite, querySuiteContents);
+                    // line to avoid command line length restrictions, particularly on windows.
-            logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
+                    const querySuitePath = `${databasePath}-queries-${type}.qls`;
-            const sarifFile = path.join(sarifFolder, `${language}.sarif`);
+                    const querySuiteContents = queries[type]
-            await codeql.databaseAnalyze(databasePath, sarifFile, querySuite, memoryFlag, addSnippetsFlag, threadsFlag);
+                        .map((q) => `- query: ${q}`)
-            logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
+                        .join("\n");
-            logger.endGroup();
+                    fs.writeFileSync(querySuitePath, querySuiteContents);
+                    logger.debug(`Query suite file for ${language}...\n${querySuiteContents}`);
+                    const sarifFile = path.join(sarifFolder, `${language}-${type}.sarif`);
+                    const codeql = codeql_1.getCodeQL(config.codeQLCmd);
+                    await codeql.databaseAnalyze(databasePath, sarifFile, querySuitePath, memoryFlag, addSnippetsFlag, threadsFlag);
+                    logger.debug(`SARIF results for database ${language} created at "${sarifFile}"`);
+                    logger.endGroup();
+                    // Record the performance
+                    const endTime = new Date().getTime();
+                    statusReport[`analyze_${type}_queries_${language}_duration_ms`] =
+                        endTime - startTime;
+                }
+            }
         }
         catch (e) {
-            // For now the fields about query performance are not populated
+            logger.error(`Error running analysis for ${language}: ${e}`);
-            return {
+            logger.info(e);
-                analyze_failure_language: language,
+            statusReport.analyze_failure_language = language;
-            };
+            return statusReport;
         }
     }
-    return {};
+    return statusReport;
 }
+exports.runQueries = runQueries;
 async function runAnalyze(repositoryNwo, commitOid, ref, analysisKey, analysisName, workflowRunID, checkoutPath, environment, githubAuth, githubUrl, doUpload, mode, outputDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logger) {
     // Delete the tracer config env var to avoid tracing ourselves
     delete process.env[sharedEnv.ODASA_TRACER_CONFIGURATION];

lib/analyze.js.map

@@ -1 +1 @@
-{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAAqC;AAErC,2CAAgD;AAGhD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAmC/B,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CACrD,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AAC3D,KAAK,UAAU,UAAU,CACvB,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,IAAI;YACF,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAC1E,uEAAuE;YACvE,2EAA2E;YAC3E,MAAM,UAAU,GAAG,GAAG,YAAY,cAAc,CAAC;YACjD,MAAM,kBAAkB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC1E,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACjD,MAAM,CAAC,KAAK,CACV,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAC7D,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,QAAQ,CAAC,CAAC;YAE9D,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,UAAU,EACV,UAAU,EACV,eAAe,EACf,WAAW,CACZ,CAAC;YAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;QAAC,OAAO,CAAC,EAAE;YACV,+DAA+D;YAC/D,OAAO;gBACL,wBAAwB,EAAE,QAAQ;aACnC,CAAC;SACH;KACF;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,MAAM,CACP,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AA5DD,gCA4DC"}
+{"version":3,"file":"analyze.js","sourceRoot":"","sources":["../src/analyze.ts"],"names":[],"mappings":";;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAE7B,gEAAkD;AAClD,qCAAqC;AAErC,2CAAgD;AAGhD,gEAAkD;AAClD,yDAA2C;AAC3C,6CAA+B;AAmC/B,KAAK,UAAU,4BAA4B,CACzC,MAA0B,EAC1B,MAAc;IAEd,sEAAsE;IACtE,oCAAoC;IACpC,aAAa,CAAC,8BAA8B,CAAC,MAAM,CAAC,CAAC;IAErD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,6BAAiB,CAAC,QAAQ,CAAC,EAAE;YAC/B,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;SACnB;KACF;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAA0B,EAC1B,MAAc;IAEd,MAAM,4BAA4B,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEnD,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC3C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CACrD,CAAC;QACF,MAAM,CAAC,QAAQ,EAAE,CAAC;KACnB;AACH,CAAC;AAED,2DAA2D;AACpD,KAAK,UAAU,UAAU,CAC9B,WAAmB,EACnB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,MAAM,YAAY,GAAwB,EAAE,CAAC;IAE7C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,MAAM,CAAC,UAAU,CAAC,aAAa,QAAQ,EAAE,CAAC,CAAC;QAE3C,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;QACzC,IAAI,OAAO,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/D,MAAM,IAAI,KAAK,CACb,qBAAqB,QAAQ,gDAAgD,CAC9E,CAAC;SACH;QAED,IAAI;YACF,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE;gBACxC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;oBAC5B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBAEvC,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAC7C,MAAM,CAAC,OAAO,EACd,QAAQ,CACT,CAAC;oBACF,uEAAuE;oBACvE,2EAA2E;oBAC3E,MAAM,cAAc,GAAG,GAAG,YAAY,YAAY,IAAI,MAAM,CAAC;oBAC7D,MAAM,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;yBACrC,GAAG,CAAC,CAAC,CAAS,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;yBACnC,IAAI,CAAC,IAAI,CAAC,CAAC;oBACd,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;oBACrD,MAAM,CAAC,KAAK,CACV,wBAAwB,QAAQ,QAAQ,kBAAkB,EAAE,CAC7D,CAAC;oBAEF,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,QAAQ,IAAI,IAAI,QAAQ,CAAC,CAAC;oBAEtE,MAAM,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;oBAC3C,MAAM,MAAM,CAAC,eAAe,CAC1B,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,CACZ,CAAC;oBAEF,MAAM,CAAC,KAAK,CACV,8BAA8B,QAAQ,gBAAgB,SAAS,GAAG,CACnE,CAAC;oBACF,MAAM,CAAC,QAAQ,EAAE,CAAC;oBAElB,yBAAyB;oBACzB,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;oBACrC,YAAY,CAAC,WAAW,IAAI,YAAY,QAAQ,cAAc,CAAC;wBAC7D,OAAO,GAAG,SAAS,CAAC;iBACvB;aACF;SACF;QAAC,OAAO,CAAC,EAAE;YACV,MAAM,CAAC,KAAK,CAAC,8BAA8B,QAAQ,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7D,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,YAAY,CAAC,wBAAwB,GAAG,QAAQ,CAAC;YACjD,OAAO,YAAY,CAAC;SACrB;KACF;IAED,OAAO,YAAY,CAAC;AACtB,CAAC;AAxED,gCAwEC;AAEM,KAAK,UAAU,UAAU,CAC9B,aAA4B,EAC5B,SAAiB,EACjB,GAAW,EACX,WAA+B,EAC/B,YAAgC,EAChC,aAAiC,EACjC,YAAoB,EACpB,WAA+B,EAC/B,UAAkB,EAClB,SAAiB,EACjB,QAAiB,EACjB,IAAe,EACf,SAAiB,EACjB,UAAkB,EAClB,eAAuB,EACvB,WAAmB,EACnB,MAA0B,EAC1B,MAAc;IAEd,8DAA8D;IAC9D,OAAO,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,CAAC,CAAC;IAEzD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE7C,MAAM,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC;IAC5C,MAAM,wBAAwB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE/C,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IAClC,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,SAAS,EACT,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,MAAM,CACP,CAAC;IAEF,IAAI,CAAC,QAAQ,EAAE;QACb,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO,EAAE,GAAG,YAAY,EAAE,CAAC;KAC5B;IAED,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CACzC,SAAS,EACT,aAAa,EACb,SAAS,EACT,GAAG,EACH,WAAW,EACX,YAAY,EACZ,aAAa,EACb,YAAY,EACZ,WAAW,EACX,UAAU,EACV,SAAS,EACT,IAAI,EACJ,MAAM,CACP,CAAC;IAEF,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,WAAW,EAAE,CAAC;AAC7C,CAAC;AA5DD,gCA4DC"}

lib/analyze.test.js

@@ -0,0 +1,63 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
+    result["default"] = mod;
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const ava_1 = __importDefault(require("ava"));
+const fs = __importStar(require("fs"));
+const analyze_1 = require("./analyze");
+const codeql_1 = require("./codeql");
+const languages_1 = require("./languages");
+const logging_1 = require("./logging");
+const testing_utils_1 = require("./testing-utils");
+const util = __importStar(require("./util"));
+testing_utils_1.setupTests(ava_1.default);
+// Checks that the duration fields are populated for the correct language
+// and correct case of builtin or custom.
+ava_1.default("status report fields", async (t) => {
+    return await util.withTmpDir(async (tmpDir) => {
+        codeql_1.setCodeQL({
+            databaseAnalyze: async () => undefined,
+        });
+        const memoryFlag = "";
+        const addSnippetsFlag = "";
+        const threadsFlag = "";
+        for (const language of Object.values(languages_1.Language)) {
+            const config = {
+                languages: [language],
+                queries: {},
+                pathsIgnore: [],
+                paths: [],
+                originalUserInput: {},
+                tempDir: tmpDir,
+                toolCacheDir: tmpDir,
+                codeQLCmd: "",
+            };
+            fs.mkdirSync(util.getCodeQLDatabasePath(config.tempDir, language), {
+                recursive: true,
+            });
+            config.queries[language] = {
+                builtin: ["foo.ql"],
+                custom: [],
+            };
+            const builtinStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logging_1.getRunnerLogger(true));
+            t.deepEqual(Object.keys(builtinStatusReport).length, 1);
+            t.true(`analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport);
+            config.queries[language] = {
+                builtin: [],
+                custom: ["foo.ql"],
+            };
+            const customStatusReport = await analyze_1.runQueries(tmpDir, memoryFlag, addSnippetsFlag, threadsFlag, config, logging_1.getRunnerLogger(true));
+            t.deepEqual(Object.keys(customStatusReport).length, 1);
+            t.true(`analyze_custom_queries_${language}_duration_ms` in customStatusReport);
+        }
+    });
+});
+//# sourceMappingURL=analyze.test.js.map

lib/analyze.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyze.test.js","sourceRoot":"","sources":["../src/analyze.test.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,8CAAuB;AACvB,uCAAyB;AAEzB,uCAAuC;AACvC,qCAAqC;AAErC,2CAAuC;AACvC,uCAA4C;AAC5C,mDAA6C;AAC7C,6CAA+B;AAE/B,0BAAU,CAAC,aAAI,CAAC,CAAC;AAEjB,yEAAyE;AACzE,yCAAyC;AACzC,aAAI,CAAC,sBAAsB,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QAC5C,kBAAS,CAAC;YACR,eAAe,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;SACvC,CAAC,CAAC;QAEH,MAAM,UAAU,GAAG,EAAE,CAAC;QACtB,MAAM,eAAe,GAAG,EAAE,CAAC;QAC3B,MAAM,WAAW,GAAG,EAAE,CAAC;QAEvB,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC,oBAAQ,CAAC,EAAE;YAC9C,MAAM,MAAM,GAAW;gBACrB,SAAS,EAAE,CAAC,QAAQ,CAAC;gBACrB,OAAO,EAAE,EAAE;gBACX,WAAW,EAAE,EAAE;gBACf,KAAK,EAAE,EAAE;gBACT,iBAAiB,EAAE,EAAE;gBACrB,OAAO,EAAE,MAAM;gBACf,YAAY,EAAE,MAAM;gBACpB,SAAS,EAAE,EAAE;aACd,CAAC;YACF,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE;gBACjE,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;YAEH,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,CAAC,QAAQ,CAAC;gBACnB,MAAM,EAAE,EAAE;aACX,CAAC;YACF,MAAM,mBAAmB,GAAG,MAAM,oBAAU,CAC1C,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC,CAAC,IAAI,CACJ,2BAA2B,QAAQ,cAAc,IAAI,mBAAmB,CACzE,CAAC;YAEF,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG;gBACzB,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,CAAC,QAAQ,CAAC;aACnB,CAAC;YACF,MAAM,kBAAkB,GAAG,MAAM,oBAAU,CACzC,MAAM,EACN,UAAU,EACV,eAAe,EACf,WAAW,EACX,MAAM,EACN,yBAAe,CAAC,IAAI,CAAC,CACtB,CAAC;YACF,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACvD,CAAC,CAAC,IAAI,CACJ,0BAA0B,QAAQ,cAAc,IAAI,kBAAkB,CACvE,CAAC;SACH;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

lib/config-utils.js

@@ -58,25 +58,39 @@ function validateQueries(resolvedQueries) {
 }
 /**
  * Run 'codeql resolve queries' and add the results to resultMap
+ *
+ * If a checkout path is given then the queries are assumed to be custom queries
+ * and an error will be thrown if there is anything invalid about the queries.
+ * If a checkout path is not given then the queries are assumed to be builtin
+ * queries, and error checking will be suppressed.
  */
-async function runResolveQueries(codeQL, resultMap, toResolve, extraSearchPath, errorOnInvalidQueries) {
+async function runResolveQueries(codeQL, resultMap, toResolve, extraSearchPath) {
     const resolvedQueries = await codeQL.resolveQueries(toResolve, extraSearchPath);
-    for (const [language, queries] of Object.entries(resolvedQueries.byLanguage)) {
+    if (extraSearchPath !== undefined) {
-        if (resultMap[language] === undefined) {
-            resultMap[language] = [];
-        }
-        resultMap[language].push(...Object.keys(queries).filter((q) => !queryIsDisabled(language, q)));
-    }
-    if (errorOnInvalidQueries) {
         validateQueries(resolvedQueries);
     }
+    for (const [language, queryPaths] of Object.entries(resolvedQueries.byLanguage)) {
+        if (resultMap[language] === undefined) {
+            resultMap[language] = {
+                builtin: [],
+                custom: [],
+            };
+        }
+        const queries = Object.keys(queryPaths).filter((q) => !queryIsDisabled(language, q));
+        if (extraSearchPath !== undefined) {
+            resultMap[language].custom.push(...queries);
+        }
+        else {
+            resultMap[language].builtin.push(...queries);
+        }
+    }
 }
 /**
  * Get the set of queries included by default.
  */
 async function addDefaultQueries(codeQL, languages, resultMap) {
     const suites = languages.map((l) => `${l}-code-scanning.qls`);
-    await runResolveQueries(codeQL, resultMap, suites, undefined, false);
+    await runResolveQueries(codeQL, resultMap, suites, undefined);
 }
 // The set of acceptable values for built-in suites from the codeql bundle
 const builtinSuites = ["security-extended", "security-and-quality"];
@@ -90,7 +104,7 @@ async function addBuiltinSuiteQueries(languages, codeQL, resultMap, suiteName, c
         throw new Error(getQueryUsesInvalid(configFile, suiteName));
     }
     const suites = languages.map((l) => `${l}-${suiteName}.qls`);
-    await runResolveQueries(codeQL, resultMap, suites, undefined, false);
+    await runResolveQueries(codeQL, resultMap, suites, undefined);
 }
 /**
  * Retrieve the set of queries at localQueryPath and add them to resultMap.
@@ -109,7 +123,7 @@ async function addLocalQueries(codeQL, resultMap, localQueryPath, checkoutPath,
     if (!(absoluteQueryPath + path.sep).startsWith(fs.realpathSync(checkoutPath) + path.sep)) {
         throw new Error(getLocalPathOutsideOfRepository(configFile, localQueryPath));
     }
-    await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath, true);
+    await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath);
 }
 /**
  * Retrieve the set of queries at the referenced remote repo and add them to resultMap.
@@ -137,7 +151,7 @@ async function addRemoteQueries(codeQL, resultMap, queryUses, tempDir, githubUrl
     const queryPath = tok.length > 2
         ? path.join(checkoutPath, tok.slice(2).join("/"))
         : checkoutPath;
-    await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
+    await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath);
 }
 /**
  * Parse a query 'uses' field to a discrete set of query files and update resultMap.
@@ -475,7 +489,9 @@ async function loadConfig(languagesInput, queriesInput, configFile, repository,
     // The list of queries should not be empty for any language. If it is then
     // it is a user configuration error.
     for (const language of languages) {
-        if (queries[language] === undefined || queries[language].length === 0) {
+        if (queries[language] === undefined ||
+            (queries[language].builtin.length === 0 &&
+                queries[language].custom.length === 0)) {
             throw new Error(`Did not detect any queries to run for ${language}. ` +
                 "Please make sure that the default queries are enabled, or you are specifying queries to run.");
         }

lib/config-utils.test.js

@@ -163,7 +163,12 @@ ava_1.default("load non-empty input", async (t) => {
         // And the config we expect it to parse to
         const expectedConfig = {
             languages: [languages_1.Language.javascript],
-            queries: { javascript: ["/foo/a.ql", "/bar/b.ql"] },
+            queries: {
+                javascript: {
+                    builtin: [],
+                    custom: ["/foo/a.ql", "/bar/b.ql"],
+                },
+            },
             pathsIgnore: ["a", "b"],
             paths: ["c/d"],
             originalUserInput: {
@@ -266,9 +271,10 @@ ava_1.default("Queries can be specified in config file", async (t) => {
         t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
         t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
         // Now check that the end result contains the default queries and the query from config
-        t.deepEqual(config.queries["javascript"].length, 2);
+        t.deepEqual(config.queries["javascript"].builtin.length, 1);
-        t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+        t.deepEqual(config.queries["javascript"].custom.length, 1);
-        t.regex(config.queries["javascript"][1], /.*\/foo$/);
+        t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
+        t.regex(config.queries["javascript"].custom[0], /.*\/foo$/);
     });
 });
 ava_1.default("Queries from config file can be overridden in workflow file", async (t) => {
@@ -298,9 +304,10 @@ ava_1.default("Queries from config file can be overridden in workflow file", asy
         t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
         t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
         // Now check that the end result contains only the default queries and the override query
-        t.deepEqual(config.queries["javascript"].length, 2);
+        t.deepEqual(config.queries["javascript"].builtin.length, 1);
-        t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+        t.deepEqual(config.queries["javascript"].custom.length, 1);
-        t.regex(config.queries["javascript"][1], /.*\/override$/);
+        t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
+        t.regex(config.queries["javascript"].custom[0], /.*\/override$/);
     });
 });
 ava_1.default("Queries in workflow file can be used in tandem with the 'disable default queries' option", async (t) => {
@@ -329,8 +336,9 @@ ava_1.default("Queries in workflow file can be used in tandem with the 'disable
         t.deepEqual(resolveQueriesArgs[0].queries.length, 1);
         t.regex(resolveQueriesArgs[0].queries[0], /.*\/workflow-query$/);
         // Now check that the end result contains only the workflow query, and not the default one
-        t.deepEqual(config.queries["javascript"].length, 1);
+        t.deepEqual(config.queries["javascript"].builtin.length, 0);
-        t.regex(config.queries["javascript"][0], /.*\/workflow-query$/);
+        t.deepEqual(config.queries["javascript"].custom.length, 1);
+        t.regex(config.queries["javascript"].custom[0], /.*\/workflow-query$/);
     });
 });
 ava_1.default("Multiple queries can be specified in workflow file, no config file required", async (t) => {
@@ -356,10 +364,11 @@ ava_1.default("Multiple queries can be specified in workflow file, no config fil
         t.regex(resolveQueriesArgs[1].queries[0], /.*\/override1$/);
         t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
         // Now check that the end result contains both the queries from the workflow, as well as the defaults
-        t.deepEqual(config.queries["javascript"].length, 3);
+        t.deepEqual(config.queries["javascript"].builtin.length, 1);
-        t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+        t.deepEqual(config.queries["javascript"].custom.length, 2);
-        t.regex(config.queries["javascript"][1], /.*\/override1$/);
+        t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
-        t.regex(config.queries["javascript"][2], /.*\/override2$/);
+        t.regex(config.queries["javascript"].custom[0], /.*\/override1$/);
+        t.regex(config.queries["javascript"].custom[1], /.*\/override2$/);
     });
 });
 ava_1.default("Queries in workflow file can be added to the set of queries without overriding config file", async (t) => {
@@ -397,11 +406,12 @@ ava_1.default("Queries in workflow file can be added to the set of queries witho
         t.deepEqual(resolveQueriesArgs[3].queries.length, 1);
         t.regex(resolveQueriesArgs[3].queries[0], /.*\/foo$/);
         // Now check that the end result contains all the queries
-        t.deepEqual(config.queries["javascript"].length, 4);
+        t.deepEqual(config.queries["javascript"].builtin.length, 1);
-        t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+        t.deepEqual(config.queries["javascript"].custom.length, 3);
-        t.regex(config.queries["javascript"][1], /.*\/additional1$/);
+        t.regex(config.queries["javascript"].builtin[0], /javascript-code-scanning.qls$/);
-        t.regex(config.queries["javascript"][2], /.*\/additional2$/);
+        t.regex(config.queries["javascript"].custom[0], /.*\/additional1$/);
-        t.regex(config.queries["javascript"][3], /.*\/foo$/);
+        t.regex(config.queries["javascript"].custom[1], /.*\/additional2$/);
+        t.regex(config.queries["javascript"].custom[2], /.*\/foo$/);
     });
 });
 ava_1.default("Invalid queries in workflow file handled correctly", async (t) => {

src/analyze.test.ts

@@ -0,0 +1,76 @@
+import test from "ava";
+import * as fs from "fs";
+
+import { runQueries } from "./analyze";
+import { setCodeQL } from "./codeql";
+import { Config } from "./config-utils";
+import { Language } from "./languages";
+import { getRunnerLogger } from "./logging";
+import { setupTests } from "./testing-utils";
+import * as util from "./util";
+
+setupTests(test);
+
+// Checks that the duration fields are populated for the correct language
+// and correct case of builtin or custom.
+test("status report fields", async (t) => {
+  return await util.withTmpDir(async (tmpDir) => {
+    setCodeQL({
+      databaseAnalyze: async () => undefined,
+    });
+
+    const memoryFlag = "";
+    const addSnippetsFlag = "";
+    const threadsFlag = "";
+
+    for (const language of Object.values(Language)) {
+      const config: Config = {
+        languages: [language],
+        queries: {},
+        pathsIgnore: [],
+        paths: [],
+        originalUserInput: {},
+        tempDir: tmpDir,
+        toolCacheDir: tmpDir,
+        codeQLCmd: "",
+      };
+      fs.mkdirSync(util.getCodeQLDatabasePath(config.tempDir, language), {
+        recursive: true,
+      });
+
+      config.queries[language] = {
+        builtin: ["foo.ql"],
+        custom: [],
+      };
+      const builtinStatusReport = await runQueries(
+        tmpDir,
+        memoryFlag,
+        addSnippetsFlag,
+        threadsFlag,
+        config,
+        getRunnerLogger(true)
+      );
+      t.deepEqual(Object.keys(builtinStatusReport).length, 1);
+      t.true(
+        `analyze_builtin_queries_${language}_duration_ms` in builtinStatusReport
+      );
+
+      config.queries[language] = {
+        builtin: [],
+        custom: ["foo.ql"],
+      };
+      const customStatusReport = await runQueries(
+        tmpDir,
+        memoryFlag,
+        addSnippetsFlag,
+        threadsFlag,
+        config,
+        getRunnerLogger(true)
+      );
+      t.deepEqual(Object.keys(customStatusReport).length, 1);
+      t.true(
+        `analyze_custom_queries_${language}_duration_ms` in customStatusReport
+      );
+    }
+  });
+});

src/analyze.ts

@@ -82,7 +82,7 @@ async function finalizeDatabaseCreation(
 }
 
 // Runs queries and creates sarif files in the given folder
-async function runQueries(
+export async function runQueries(
   sarifFolder: string,
   memoryFlag: string,
   addSnippetsFlag: string,
@@ -90,52 +90,70 @@ async function runQueries(
   config: configUtils.Config,
   logger: Logger
 ): Promise<QueriesStatusReport> {
-  const codeql = getCodeQL(config.codeQLCmd);
+  const statusReport: QueriesStatusReport = {};
+
   for (const language of config.languages) {
     logger.startGroup(`Analyzing ${language}`);
 
-    const queries = config.queries[language] || [];
+    const queries = config.queries[language];
-    if (queries.length === 0) {
+    if (queries.builtin.length === 0 && queries.custom.length === 0) {
       throw new Error(
         `Unable to analyse ${language} as no queries were selected for this language`
       );
     }
 
     try {
-      const databasePath = util.getCodeQLDatabasePath(config.tempDir, language);
+      for (const type of ["builtin", "custom"]) {
-      // Pass the queries to codeql using a file instead of using the command
+        if (queries[type].length > 0) {
-      // line to avoid command line length restrictions, particularly on windows.
+          const startTime = new Date().getTime();
-      const querySuite = `${databasePath}-queries.qls`;
-      const querySuiteContents = queries.map((q) => `- query: ${q}`).join("\n");
-      fs.writeFileSync(querySuite, querySuiteContents);
-      logger.debug(
-        `Query suite file for ${language}...\n${querySuiteContents}`
-      );
 
-      const sarifFile = path.join(sarifFolder, `${language}.sarif`);
+          const databasePath = util.getCodeQLDatabasePath(
+            config.tempDir,
+            language
+          );
+          // Pass the queries to codeql using a file instead of using the command
+          // line to avoid command line length restrictions, particularly on windows.
+          const querySuitePath = `${databasePath}-queries-${type}.qls`;
+          const querySuiteContents = queries[type]
+            .map((q: string) => `- query: ${q}`)
+            .join("\n");
+          fs.writeFileSync(querySuitePath, querySuiteContents);
+          logger.debug(
+            `Query suite file for ${language}...\n${querySuiteContents}`
+          );
 
-      await codeql.databaseAnalyze(
+          const sarifFile = path.join(sarifFolder, `${language}-${type}.sarif`);
-        databasePath,
-        sarifFile,
-        querySuite,
-        memoryFlag,
-        addSnippetsFlag,
-        threadsFlag
-      );
 
-      logger.debug(
+          const codeql = getCodeQL(config.codeQLCmd);
-        `SARIF results for database ${language} created at "${sarifFile}"`
+          await codeql.databaseAnalyze(
-      );
+            databasePath,
-      logger.endGroup();
+            sarifFile,
+            querySuitePath,
+            memoryFlag,
+            addSnippetsFlag,
+            threadsFlag
+          );
+
+          logger.debug(
+            `SARIF results for database ${language} created at "${sarifFile}"`
+          );
+          logger.endGroup();
+
+          // Record the performance
+          const endTime = new Date().getTime();
+          statusReport[`analyze_${type}_queries_${language}_duration_ms`] =
+            endTime - startTime;
+        }
+      }
     } catch (e) {
-      // For now the fields about query performance are not populated
+      logger.error(`Error running analysis for ${language}: ${e}`);
-      return {
+      logger.info(e);
-        analyze_failure_language: language,
+      statusReport.analyze_failure_language = language;
-      };
+      return statusReport;
     }
   }
 
-  return {};
+  return statusReport;
 }
 
 export async function runAnalyze(

src/config-utils.test.ts

@@ -272,7 +272,12 @@ test("load non-empty input", async (t) => {
     // And the config we expect it to parse to
     const expectedConfig: configUtils.Config = {
       languages: [Language.javascript],
-      queries: { javascript: ["/foo/a.ql", "/bar/b.ql"] },
+      queries: {
+        javascript: {
+          builtin: [],
+          custom: ["/foo/a.ql", "/bar/b.ql"],
+        },
+      },
       pathsIgnore: ["a", "b"],
       paths: ["c/d"],
       originalUserInput: {
@@ -442,9 +447,13 @@ test("Queries can be specified in config file", async (t) => {
     t.regex(resolveQueriesArgs[1].queries[0], /.*\/foo$/);
 
     // Now check that the end result contains the default queries and the query from config
-    t.deepEqual(config.queries["javascript"].length, 2);
+    t.deepEqual(config.queries["javascript"].builtin.length, 1);
-    t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+    t.deepEqual(config.queries["javascript"].custom.length, 1);
-    t.regex(config.queries["javascript"][1], /.*\/foo$/);
+    t.regex(
+      config.queries["javascript"].builtin[0],
+      /javascript-code-scanning.qls$/
+    );
+    t.regex(config.queries["javascript"].custom[0], /.*\/foo$/);
   });
 });
 
@@ -501,9 +510,13 @@ test("Queries from config file can be overridden in workflow file", async (t) =>
     t.regex(resolveQueriesArgs[1].queries[0], /.*\/override$/);
 
     // Now check that the end result contains only the default queries and the override query
-    t.deepEqual(config.queries["javascript"].length, 2);
+    t.deepEqual(config.queries["javascript"].builtin.length, 1);
-    t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+    t.deepEqual(config.queries["javascript"].custom.length, 1);
-    t.regex(config.queries["javascript"][1], /.*\/override$/);
+    t.regex(
+      config.queries["javascript"].builtin[0],
+      /javascript-code-scanning.qls$/
+    );
+    t.regex(config.queries["javascript"].custom[0], /.*\/override$/);
   });
 });
 
@@ -558,8 +571,9 @@ test("Queries in workflow file can be used in tandem with the 'disable default q
     t.regex(resolveQueriesArgs[0].queries[0], /.*\/workflow-query$/);
 
     // Now check that the end result contains only the workflow query, and not the default one
-    t.deepEqual(config.queries["javascript"].length, 1);
+    t.deepEqual(config.queries["javascript"].builtin.length, 0);
-    t.regex(config.queries["javascript"][0], /.*\/workflow-query$/);
+    t.deepEqual(config.queries["javascript"].custom.length, 1);
+    t.regex(config.queries["javascript"].custom[0], /.*\/workflow-query$/);
   });
 });
 
@@ -610,10 +624,14 @@ test("Multiple queries can be specified in workflow file, no config file require
     t.regex(resolveQueriesArgs[2].queries[0], /.*\/override2$/);
 
     // Now check that the end result contains both the queries from the workflow, as well as the defaults
-    t.deepEqual(config.queries["javascript"].length, 3);
+    t.deepEqual(config.queries["javascript"].builtin.length, 1);
-    t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+    t.deepEqual(config.queries["javascript"].custom.length, 2);
-    t.regex(config.queries["javascript"][1], /.*\/override1$/);
+    t.regex(
-    t.regex(config.queries["javascript"][2], /.*\/override2$/);
+      config.queries["javascript"].builtin[0],
+      /javascript-code-scanning.qls$/
+    );
+    t.regex(config.queries["javascript"].custom[0], /.*\/override1$/);
+    t.regex(config.queries["javascript"].custom[1], /.*\/override2$/);
   });
 });
 
@@ -678,11 +696,15 @@ test("Queries in workflow file can be added to the set of queries without overri
     t.regex(resolveQueriesArgs[3].queries[0], /.*\/foo$/);
 
     // Now check that the end result contains all the queries
-    t.deepEqual(config.queries["javascript"].length, 4);
+    t.deepEqual(config.queries["javascript"].builtin.length, 1);
-    t.regex(config.queries["javascript"][0], /javascript-code-scanning.qls$/);
+    t.deepEqual(config.queries["javascript"].custom.length, 3);
-    t.regex(config.queries["javascript"][1], /.*\/additional1$/);
+    t.regex(
-    t.regex(config.queries["javascript"][2], /.*\/additional2$/);
+      config.queries["javascript"].builtin[0],
-    t.regex(config.queries["javascript"][3], /.*\/foo$/);
+      /javascript-code-scanning.qls$/
+    );
+    t.regex(config.queries["javascript"].custom[0], /.*\/additional1$/);
+    t.regex(config.queries["javascript"].custom[1], /.*\/additional2$/);
+    t.regex(config.queries["javascript"].custom[2], /.*\/foo$/);
   });
 });
 

src/config-utils.ts

@@ -31,6 +31,24 @@ export interface UserConfig {
   paths?: string[];
 }
 
+/**
+ * Lists of query files for each language.
+ * Will only contain .ql files and not other kinds of files,
+ * and all file paths will be absolute.
+ *
+ * The queries are split between ones from a builtin suite
+ * and custom queries from unknown locations. This allows us to treat
+ * them separately if we want to, for example to measure performance.
+ */
+type Queries = {
+  [language: string]: {
+    /** Queries from one of the builtin suites */
+    builtin: string[];
+    /** Custom queries, from a non-standard location */
+    custom: string[];
+  };
+};
+
 /**
  * Format of the parsed config file.
  */
@@ -41,10 +59,8 @@ export interface Config {
   languages: Language[];
   /**
    * Map from language to query files.
-   * Will only contain .ql files and not other kinds of files,
-   * and all file paths will be absolute.
    */
-  queries: { [language: string]: string[] };
+  queries: Queries;
   /**
    * List of paths to ignore from analysis.
    */
@@ -131,32 +147,44 @@ function validateQueries(resolvedQueries: ResolveQueriesOutput) {
 
 /**
  * Run 'codeql resolve queries' and add the results to resultMap
+ *
+ * If a checkout path is given then the queries are assumed to be custom queries
+ * and an error will be thrown if there is anything invalid about the queries.
+ * If a checkout path is not given then the queries are assumed to be builtin
+ * queries, and error checking will be suppressed.
  */
 async function runResolveQueries(
   codeQL: CodeQL,
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   toResolve: string[],
-  extraSearchPath: string | undefined,
+  extraSearchPath: string | undefined
-  errorOnInvalidQueries: boolean
 ) {
   const resolvedQueries = await codeQL.resolveQueries(
     toResolve,
     extraSearchPath
   );
 
-  for (const [language, queries] of Object.entries(
+  if (extraSearchPath !== undefined) {
+    validateQueries(resolvedQueries);
+  }
+
+  for (const [language, queryPaths] of Object.entries(
     resolvedQueries.byLanguage
   )) {
     if (resultMap[language] === undefined) {
-      resultMap[language] = [];
+      resultMap[language] = {
+        builtin: [],
+        custom: [],
+      };
     }
-    resultMap[language].push(
+    const queries = Object.keys(queryPaths).filter(
-      ...Object.keys(queries).filter((q) => !queryIsDisabled(language, q))
+      (q) => !queryIsDisabled(language, q)
     );
-  }
+    if (extraSearchPath !== undefined) {
-
+      resultMap[language].custom.push(...queries);
-  if (errorOnInvalidQueries) {
+    } else {
-    validateQueries(resolvedQueries);
+      resultMap[language].builtin.push(...queries);
+    }
   }
 }
 
@@ -166,10 +194,10 @@ async function runResolveQueries(
 async function addDefaultQueries(
   codeQL: CodeQL,
   languages: string[],
-  resultMap: { [language: string]: string[] }
+  resultMap: Queries
 ) {
   const suites = languages.map((l) => `${l}-code-scanning.qls`);
-  await runResolveQueries(codeQL, resultMap, suites, undefined, false);
+  await runResolveQueries(codeQL, resultMap, suites, undefined);
 }
 
 // The set of acceptable values for built-in suites from the codeql bundle
@@ -182,7 +210,7 @@ const builtinSuites = ["security-extended", "security-and-quality"] as const;
 async function addBuiltinSuiteQueries(
   languages: string[],
   codeQL: CodeQL,
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   suiteName: string,
   configFile?: string
 ) {
@@ -192,7 +220,7 @@ async function addBuiltinSuiteQueries(
   }
 
   const suites = languages.map((l) => `${l}-${suiteName}.qls`);
-  await runResolveQueries(codeQL, resultMap, suites, undefined, false);
+  await runResolveQueries(codeQL, resultMap, suites, undefined);
 }
 
 /**
@@ -200,7 +228,7 @@ async function addBuiltinSuiteQueries(
  */
 async function addLocalQueries(
   codeQL: CodeQL,
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   localQueryPath: string,
   checkoutPath: string,
   configFile?: string
@@ -228,13 +256,7 @@ async function addLocalQueries(
     );
   }
 
-  await runResolveQueries(
+  await runResolveQueries(codeQL, resultMap, [absoluteQueryPath], checkoutPath);
-    codeQL,
-    resultMap,
-    [absoluteQueryPath],
-    checkoutPath,
-    true
-  );
 }
 
 /**
@@ -242,7 +264,7 @@ async function addLocalQueries(
  */
 async function addRemoteQueries(
   codeQL: CodeQL,
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   queryUses: string,
   tempDir: string,
   githubUrl: string,
@@ -283,7 +305,7 @@ async function addRemoteQueries(
       ? path.join(checkoutPath, tok.slice(2).join("/"))
       : checkoutPath;
 
-  await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath, true);
+  await runResolveQueries(codeQL, resultMap, [queryPath], checkoutPath);
 }
 
 /**
@@ -297,7 +319,7 @@ async function addRemoteQueries(
 async function parseQueryUses(
   languages: string[],
   codeQL: CodeQL,
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   queryUses: string,
   tempDir: string,
   checkoutPath: string,
@@ -660,7 +682,7 @@ async function addQueriesFromWorkflow(
   codeQL: CodeQL,
   queriesInput: string,
   languages: string[],
-  resultMap: { [language: string]: string[] },
+  resultMap: Queries,
   tempDir: string,
   checkoutPath: string,
   githubUrl: string,
@@ -718,7 +740,7 @@ export async function getDefaultConfig(
     githubUrl,
     logger
   );
-  const queries = {};
+  const queries: Queries = {};
   await addDefaultQueries(codeQL, languages, queries);
   if (queriesInput) {
     await addQueriesFromWorkflow(
@@ -790,7 +812,7 @@ async function loadConfig(
     logger
   );
 
-  const queries = {};
+  const queries: Queries = {};
   const pathsIgnore: string[] = [];
   const paths: string[] = [];
 
@@ -880,7 +902,11 @@ async function loadConfig(
   // The list of queries should not be empty for any language. If it is then
   // it is a user configuration error.
   for (const language of languages) {
-    if (queries[language] === undefined || queries[language].length === 0) {
+    if (
+      queries[language] === undefined ||
+      (queries[language].builtin.length === 0 &&
+        queries[language].custom.length === 0)
+    ) {
       throw new Error(
         `Did not detect any queries to run for ${language}. ` +
           "Please make sure that the default queries are enabled, or you are specifying queries to run."