Merge pull request #1219 from github/angelapwen/autobuild-in-analyze

Autobuild Go in `analyze` if not already built

.github/workflows/__go-reconciled-tracing-legacy-workflow.yml

@@ -0,0 +1,88 @@
+# Warning: This file is generated automatically, and should not be modified.
+# Instead, please modify the template in the pr-checks directory and run:
+#     pip install ruamel.yaml && python3 sync.py
+# to regenerate this file.
+
+name: 'PR Check - Go: Reconciled tracing with legacy workflow'
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GO111MODULE: auto
+on:
+  push:
+    branches:
+    - main
+    - releases/v1
+    - releases/v2
+  pull_request:
+    types:
+    - opened
+    - synchronize
+    - reopened
+    - ready_for_review
+  workflow_dispatch: {}
+jobs:
+  go-reconciled-tracing-legacy-workflow:
+    strategy:
+      matrix:
+        include:
+        - os: ubuntu-latest
+          version: stable-20210308
+        - os: macos-latest
+          version: stable-20210308
+        - os: ubuntu-latest
+          version: stable-20210319
+        - os: macos-latest
+          version: stable-20210319
+        - os: ubuntu-latest
+          version: stable-20210809
+        - os: macos-latest
+          version: stable-20210809
+        - os: ubuntu-latest
+          version: cached
+        - os: macos-latest
+          version: cached
+        - os: ubuntu-latest
+          version: latest
+        - os: macos-latest
+          version: latest
+        - os: ubuntu-latest
+          version: nightly-latest
+        - os: macos-latest
+          version: nightly-latest
+    name: 'Go: Reconciled tracing with legacy workflow'
+    timeout-minutes: 45
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v3
+    - name: Prepare test
+      id: prepare-test
+      uses: ./.github/prepare-test
+      with:
+        version: ${{ matrix.version }}
+    - uses: actions/setup-go@v3
+      with:
+        go-version: ^1.13.1
+    - uses: ./../action/init
+      with:
+        languages: go
+        tools: ${{ steps.prepare-test.outputs.tools-url }}
+      env:
+        TEST_MODE: true
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+    - shell: bash
+      run: |
+        cd "$RUNNER_TEMP/codeql_databases"
+        if [[ ! -d go ]]; then
+          echo "Did not find a Go database"
+          exit 1
+        fi
+    env:
+  # Enable reconciled Go tracing beta functionality
+      CODEQL_ACTION_RECONCILE_GO_EXTRACTION: 'true'
+  # Remove this once we start injecting the Go tracer in the init step.
+      CODEQL_EXTRACTOR_GO_BUILD_TRACING: on
+      DOTNET_GENERATE_ASPNET_CERTIFICATE: 'false'
+      INTERNAL_CODEQL_ACTION_DEBUG_LOC: true

lib/analyze-action.js

@@ -18,18 +18,25 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runPromise = exports.sendStatusReport = void 0;
+const fs = __importStar(require("fs"));
+const path_1 = __importDefault(require("path"));
 // We need to import `performance` on Node 12
 const perf_hooks_1 = require("perf_hooks");
 const core = __importStar(require("@actions/core"));
 const actionsUtil = __importStar(require("./actions-util"));
 const analyze_1 = require("./analyze");
 const api_client_1 = require("./api-client");
+const autobuild_1 = require("./autobuild");
 const codeql_1 = require("./codeql");
 const config_utils_1 = require("./config-utils");
 const database_upload_1 = require("./database-upload");
 const feature_flags_1 = require("./feature-flags");
+const languages_1 = require("./languages");
 const logging_1 = require("./logging");
 const repository_1 = require("./repository");
 const trap_caching_1 = require("./trap-caching");
@@ -68,6 +75,62 @@ function hasBadExpectErrorInput() {
     return (actionsUtil.getOptionalInput("expect-error") !== "false" &&
         !util.isInTestMode());
 }
+/**
+ * Returns whether any TRAP files exist under the `db-go` folder,
+ * indicating whether Go extraction has extracted at least one file.
+ */
+function doesGoExtractionOutputExist(config) {
+    const golangDbDirectory = util.getCodeQLDatabasePath(config, languages_1.Language.go);
+    const trapDirectory = path_1.default.join(golangDbDirectory, "trap", languages_1.Language.go);
+    return (fs.existsSync(trapDirectory) &&
+        fs
+            .readdirSync(trapDirectory)
+            .some((fileName) => [
+            ".trap",
+            ".trap.gz",
+            ".trap.br",
+            ".trap.tar.gz",
+            ".trap.tar.br",
+            ".trap.tar",
+        ].some((ext) => fileName.endsWith(ext))));
+}
+/**
+ * When Go extraction reconciliation is enabled, either via the feature flag
+ * or an environment variable, we will attempt to autobuild Go to preserve
+ * compatibility for users who have set up Go using a legacy scanning style
+ * CodeQL workflow, i.e. one without an autobuild step or manual build
+ * steps.
+ *
+ * - We detect whether an autobuild step is present by checking the
+ * `CODEQL_ACTION_DID_AUTOBUILD_GOLANG` environment variable, which is set
+ * when the autobuilder is invoked.
+ * - We approximate whether manual build steps are present by looking at
+ * whether any extraction output already exists for Go.
+ */
+async function runAutobuildIfLegacyGoWorkflow(config, featureFlags, logger) {
+    if (!config.languages.includes(languages_1.Language.go)) {
+        return;
+    }
+    if (process.env["CODEQL_ACTION_RECONCILE_GO_EXTRACTION"] !== "true" &&
+        !(await featureFlags.getValue(feature_flags_1.FeatureFlag.GolangExtractionReconciliationEnabled))) {
+        logger.debug("Won't run Go autobuild since Go extraction reconciliation is not enabled.");
+        return;
+    }
+    if (process.env["CODEQL_ACTION_DID_AUTOBUILD_GOLANG"] === "true") {
+        // This log line is info level while Go extraction reconciliation is in beta.
+        // We will make it debug level once Go extraction reconciliation is GA.
+        logger.info("Won't run Go autobuild since it has already been run.");
+        return;
+    }
+    // This captures whether a user has added manual build steps for Go
+    if (doesGoExtractionOutputExist(config)) {
+        // This log line is info level while Go extraction reconciliation is in beta.
+        // We will make it debug level once Go extraction reconciliation is GA.
+        logger.info("Won't run Go autobuild since at least one file of Go code has already been extracted.");
+        return;
+    }
+    await (0, autobuild_1.runAutobuild)(languages_1.Language.go, config, logger);
+}
 async function run() {
     const startedAt = new Date();
     let uploadResult = undefined;
@@ -102,6 +165,7 @@ async function run() {
         const repositoryNwo = (0, repository_1.parseRepositoryNwo)(util.getRequiredEnvParam("GITHUB_REPOSITORY"));
         const gitHubVersion = await (0, api_client_1.getGitHubVersionActionsOnly)();
         const featureFlags = new feature_flags_1.GitHubFeatureFlags(gitHubVersion, apiDetails, repositoryNwo, logger);
+        await runAutobuildIfLegacyGoWorkflow(config, featureFlags, logger);
         dbCreationTimings = await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger, featureFlags);
         if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
             runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);

lib/autobuild-action.js

@@ -23,6 +23,7 @@ const core = __importStar(require("@actions/core"));
 const actions_util_1 = require("./actions-util");
 const autobuild_1 = require("./autobuild");
 const config_utils = __importStar(require("./config-utils"));
+const languages_1 = require("./languages");
 const logging_1 = require("./logging");
 const util_1 = require("./util");
 // eslint-disable-next-line import/no-commonjs
@@ -59,6 +60,9 @@ async function run() {
                 process.chdir(workingDirectory);
             }
             await (0, autobuild_1.runAutobuild)(language, config, logger);
+            if (language === languages_1.Language.go) {
+                core.exportVariable("CODEQL_ACTION_DID_AUTOBUILD_GOLANG", "true");
+            }
         }
     }
     catch (error) {

lib/autobuild-action.js.map

@@ -1 +1 @@
-{"version":3,"file":"autobuild-action.js","sourceRoot":"","sources":["../src/autobuild-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAOwB;AACxB,2CAAuE;AACvE,6DAA+C;AAE/C,uCAA6C;AAC7C,iCAAyE;AAEzE,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AASvC,KAAK,UAAU,yBAAyB,CACtC,SAAe,EACf,YAAsB,EACtB,eAAwB,EACxB,KAAa;IAEb,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,IAAA,+BAAgB,EAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,MAAM,IAAA,qCAAsB,EACnD,WAAW,EACX,MAAM,EACN,SAAS,EACT,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,EACd,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAA0B;QAC1C,GAAG,gBAAgB;QACnB,mBAAmB,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3C,iBAAiB,EAAE,eAAe;KACnC,CAAC;IACF,MAAM,IAAA,+BAAgB,EAAC,YAAY,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAClC,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,QAAQ,GAAyB,SAAS,CAAC;IAC/C,IAAI;QACF,IACE,CAAC,CAAC,MAAM,IAAA,+BAAgB,EACtB,MAAM,IAAA,qCAAsB,EAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CACjE,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,IAAA,oCAAqB,GAAE,EACvB,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,QAAQ,GAAG,IAAA,sCAA0B,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;YAC/D,IAAI,gBAAgB,EAAE;gBACpB,MAAM,CAAC,IAAI,CACT,6CAA6C,gBAAgB,EAAE,CAChE,CAAC;gBACF,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACjC;YACD,MAAM,IAAA,wBAAY,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;SAC9C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CACZ,mIACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,yBAAyB,CAC7B,SAAS,EACT,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,EAC1B,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACF,OAAO;KACR;IAED,MAAM,yBAAyB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}
+{"version":3,"file":"autobuild-action.js","sourceRoot":"","sources":["../src/autobuild-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,iDAOwB;AACxB,2CAAuE;AACvE,6DAA+C;AAC/C,2CAAuC;AACvC,uCAA6C;AAC7C,iCAAyE;AAEzE,8CAA8C;AAC9C,MAAM,GAAG,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;AASvC,KAAK,UAAU,yBAAyB,CACtC,SAAe,EACf,YAAsB,EACtB,eAAwB,EACxB,KAAa;IAEb,IAAA,4BAAqB,EAAC,WAAI,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;IAEjD,MAAM,MAAM,GAAG,IAAA,+BAAgB,EAAC,KAAK,EAAE,eAAe,CAAC,CAAC;IACxD,MAAM,gBAAgB,GAAG,MAAM,IAAA,qCAAsB,EACnD,WAAW,EACX,MAAM,EACN,SAAS,EACT,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,OAAO,EACd,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,CACb,CAAC;IACF,MAAM,YAAY,GAA0B;QAC1C,GAAG,gBAAgB;QACnB,mBAAmB,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC;QAC3C,iBAAiB,EAAE,eAAe;KACnC,CAAC;IACF,MAAM,IAAA,+BAAgB,EAAC,YAAY,CAAC,CAAC;AACvC,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,MAAM,MAAM,GAAG,IAAA,0BAAgB,GAAE,CAAC;IAClC,MAAM,IAAA,yBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACtC,IAAI,QAAQ,GAAyB,SAAS,CAAC;IAC/C,IAAI;QACF,IACE,CAAC,CAAC,MAAM,IAAA,+BAAgB,EACtB,MAAM,IAAA,qCAAsB,EAAC,WAAW,EAAE,UAAU,EAAE,SAAS,CAAC,CACjE,CAAC,EACF;YACA,OAAO;SACR;QAED,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,IAAA,oCAAqB,GAAE,EACvB,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,QAAQ,GAAG,IAAA,sCAA0B,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,MAAM,gBAAgB,GAAG,IAAA,+BAAgB,EAAC,mBAAmB,CAAC,CAAC;YAC/D,IAAI,gBAAgB,EAAE;gBACpB,MAAM,CAAC,IAAI,CACT,6CAA6C,gBAAgB,EAAE,CAChE,CAAC;gBACF,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;aACjC;YACD,MAAM,IAAA,wBAAY,EAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;YAC7C,IAAI,QAAQ,KAAK,oBAAQ,CAAC,EAAE,EAAE;gBAC5B,IAAI,CAAC,cAAc,CAAC,oCAAoC,EAAE,MAAM,CAAC,CAAC;aACnE;SACF;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CACZ,mIACE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CACvD,EAAE,CACH,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,MAAM,yBAAyB,CAC7B,SAAS,EACT,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,EAC1B,QAAQ,EACR,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACF,OAAO;KACR;IAED,MAAM,yBAAyB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,4BAA4B,KAAK,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/feature-flags.js

@@ -28,6 +28,7 @@ var FeatureFlag;
     FeatureFlag["LuaTracerConfigEnabled"] = "lua_tracer_config_enabled";
     FeatureFlag["MlPoweredQueriesEnabled"] = "ml_powered_queries_enabled";
     FeatureFlag["TrapCachingEnabled"] = "trap_caching_enabled";
+    FeatureFlag["GolangExtractionReconciliationEnabled"] = "golang_extraction_reconciliation_enabled";
 })(FeatureFlag = exports.FeatureFlag || (exports.FeatureFlag = {}));
 class GitHubFeatureFlags {
     constructor(gitHubVersion, apiDetails, repositoryNwo, logger) {

lib/feature-flags.js.map

@@ -1 +1 @@
-{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAG9D,6CAA+B;AAM/B,IAAY,WAKX;AALD,WAAY,WAAW;IACrB,kEAAmD,CAAA;IACnD,mEAAoD,CAAA;IACpD,qEAAsD,CAAA;IACtD,0DAA2C,CAAA;AAC7C,CAAC,EALW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAKtB;AAUD,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,aAA4B,EAC5B,MAAc;QAHd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,IAAiB;QAC9B,oDAAoD;QACpD,IAAI,IAAI,KAAK,WAAW,CAAC,sBAAsB,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE;YACtE,OAAO,KAAK,CAAC;SACd;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qCAAqC,IAAI,4BAA4B,CACtE,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;oBAC/B,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;iBAC9B,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;oBAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gGAAgG;wBAC9F,oEAAoE;wBACpE,qFAAqF;wBACrF,kFAAkF,CAAC,EAAE,CACxF,CAAC;iBACH;qBAAM;oBACL,uFAAuF;oBACvF,mFAAmF;oBACnF,2FAA2F;oBAC3F,qBAAqB;oBACrB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAAC,EAAE,CAChE,CAAC;iBACH;aACF;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AA5ED,gDA4EC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAAC,YAA2B;IAC5D,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;AACJ,CAAC;AAND,gDAMC"}
+{"version":3,"file":"feature-flags.js","sourceRoot":"","sources":["../src/feature-flags.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;AAAA,6CAA8D;AAG9D,6CAA+B;AAM/B,IAAY,WAMX;AAND,WAAY,WAAW;IACrB,kEAAmD,CAAA;IACnD,mEAAoD,CAAA;IACpD,qEAAsD,CAAA;IACtD,0DAA2C,CAAA;IAC3C,iGAAkF,CAAA;AACpF,CAAC,EANW,WAAW,GAAX,mBAAW,KAAX,mBAAW,QAMtB;AAUD,MAAa,kBAAkB;IAG7B,YACU,aAAiC,EACjC,UAA4B,EAC5B,aAA4B,EAC5B,MAAc;QAHd,kBAAa,GAAb,aAAa,CAAoB;QACjC,eAAU,GAAV,UAAU,CAAkB;QAC5B,kBAAa,GAAb,aAAa,CAAe;QAC5B,WAAM,GAAN,MAAM,CAAQ;IACrB,CAAC;IAEJ,KAAK,CAAC,QAAQ,CAAC,IAAiB;QAC9B,oDAAoD;QACpD,IAAI,IAAI,KAAK,WAAW,CAAC,sBAAsB,IAAI,IAAI,CAAC,YAAY,EAAE,EAAE;YACtE,OAAO,KAAK,CAAC;SACd;QAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC7C,IAAI,QAAQ,KAAK,SAAS,EAAE;YAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,qCAAqC,IAAI,4BAA4B,CACtE,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,SAAS,KAAK,SAAS,EAAE;YAC3B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,iBAAiB,IAAI,uDAAuD,CAC7E,CAAC;YACF,OAAO,KAAK,CAAC;SACd;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,iDAAiD;YACjD,IAAI,IAAI,CAAC,aAAa,CAAC,IAAI,KAAK,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE;gBACzD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,8DAA8D,CAC/D,CAAC;gBACF,OAAO,EAAE,CAAC;aACX;YACD,MAAM,MAAM,GAAG,IAAA,yBAAY,EAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC7C,IAAI;gBACF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,OAAO,CACnC,8DAA8D,EAC9D;oBACE,KAAK,EAAE,IAAI,CAAC,aAAa,CAAC,KAAK;oBAC/B,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI;iBAC9B,CACF,CAAC;gBACF,OAAO,QAAQ,CAAC,IAAI,CAAC;aACtB;YAAC,OAAO,CAAC,EAAE;gBACV,IAAI,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE;oBAC3C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gGAAgG;wBAC9F,oEAAoE;wBACpE,qFAAqF;wBACrF,kFAAkF,CAAC,EAAE,CACxF,CAAC;iBACH;qBAAM;oBACL,uFAAuF;oBACvF,mFAAmF;oBACnF,2FAA2F;oBAC3F,qBAAqB;oBACrB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAAC,EAAE,CAChE,CAAC;iBACH;aACF;QACH,CAAC,CAAC;QAEF,MAAM,WAAW,GAAG,IAAI,CAAC,iBAAiB,IAAI,CAAC,MAAM,eAAe,EAAE,CAAC,CAAC;QACxE,IAAI,CAAC,iBAAiB,GAAG,WAAW,CAAC;QACrC,OAAO,WAAW,CAAC;IACrB,CAAC;CACF;AA5ED,gDA4EC;AAED;;;;GAIG;AACH,SAAgB,kBAAkB,CAAC,YAA2B;IAC5D,OAAO;QACL,QAAQ,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,OAAO,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,CAAC;KACF,CAAC;AACJ,CAAC;AAND,gDAMC"}

pr-checks/checks/go-reconciled-tracing-legacy-workflow.yml

@@ -0,0 +1,29 @@
+name: "Go: Reconciled tracing with legacy workflow"
+description: "Checks that we run the autobuilder in legacy workflows with neither an autobuild step nor manual build steps"
+os: ["ubuntu-latest", "macos-latest"]
+env:
+  # Enable reconciled Go tracing beta functionality
+  CODEQL_ACTION_RECONCILE_GO_EXTRACTION: "true"
+  # Remove this once we start injecting the Go tracer in the init step.
+  CODEQL_EXTRACTOR_GO_BUILD_TRACING: "on"
+  DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
+steps:
+  - uses: actions/setup-go@v3
+    with:
+      go-version: "^1.13.1"
+  - uses: ./../action/init
+    with:
+      languages: go
+      tools: ${{ steps.prepare-test.outputs.tools-url }}
+    env:
+      TEST_MODE: true
+  - uses: ./../action/analyze
+    env:
+      TEST_MODE: true
+  - shell: bash
+    run: |
+      cd "$RUNNER_TEMP/codeql_databases"
+      if [[ ! -d go ]]; then
+        echo "Did not find a Go database"
+        exit 1
+      fi

src/analyze-action.ts

@@ -1,3 +1,5 @@
+import * as fs from "fs";
+import path from "path";
 // We need to import `performance` on Node 12
 import { performance } from "perf_hooks";
 
@@ -13,10 +15,12 @@ import {
   runQueries,
 } from "./analyze";
 import { getGitHubVersionActionsOnly } from "./api-client";
+import { runAutobuild } from "./autobuild";
 import { getCodeQL } from "./codeql";
 import { Config, getConfig } from "./config-utils";
 import { uploadDatabases } from "./database-upload";
-import { GitHubFeatureFlags } from "./feature-flags";
+import { FeatureFlag, FeatureFlags, GitHubFeatureFlags } from "./feature-flags";
+import { Language } from "./languages";
 import { getActionsLogger, Logger } from "./logging";
 import { parseRepositoryNwo } from "./repository";
 import { getTotalCacheSize, uploadTrapCaches } from "./trap-caching";
@@ -97,6 +101,80 @@ function hasBadExpectErrorInput(): boolean {
   );
 }
 
+/**
+ * Returns whether any TRAP files exist under the `db-go` folder,
+ * indicating whether Go extraction has extracted at least one file.
+ */
+function doesGoExtractionOutputExist(config: Config): boolean {
+  const golangDbDirectory = util.getCodeQLDatabasePath(config, Language.go);
+  const trapDirectory = path.join(golangDbDirectory, "trap", Language.go);
+  return (
+    fs.existsSync(trapDirectory) &&
+    fs
+      .readdirSync(trapDirectory)
+      .some((fileName) =>
+        [
+          ".trap",
+          ".trap.gz",
+          ".trap.br",
+          ".trap.tar.gz",
+          ".trap.tar.br",
+          ".trap.tar",
+        ].some((ext) => fileName.endsWith(ext))
+      )
+  );
+}
+
+/**
+ * When Go extraction reconciliation is enabled, either via the feature flag
+ * or an environment variable, we will attempt to autobuild Go to preserve
+ * compatibility for users who have set up Go using a legacy scanning style
+ * CodeQL workflow, i.e. one without an autobuild step or manual build
+ * steps.
+ *
+ * - We detect whether an autobuild step is present by checking the
+ * `CODEQL_ACTION_DID_AUTOBUILD_GOLANG` environment variable, which is set
+ * when the autobuilder is invoked.
+ * - We approximate whether manual build steps are present by looking at
+ * whether any extraction output already exists for Go.
+ */
+async function runAutobuildIfLegacyGoWorkflow(
+  config: Config,
+  featureFlags: FeatureFlags,
+  logger: Logger
+) {
+  if (!config.languages.includes(Language.go)) {
+    return;
+  }
+  if (
+    process.env["CODEQL_ACTION_RECONCILE_GO_EXTRACTION"] !== "true" &&
+    !(await featureFlags.getValue(
+      FeatureFlag.GolangExtractionReconciliationEnabled
+    ))
+  ) {
+    logger.debug(
+      "Won't run Go autobuild since Go extraction reconciliation is not enabled."
+    );
+    return;
+  }
+  if (process.env["CODEQL_ACTION_DID_AUTOBUILD_GOLANG"] === "true") {
+    // This log line is info level while Go extraction reconciliation is in beta.
+    // We will make it debug level once Go extraction reconciliation is GA.
+    logger.info("Won't run Go autobuild since it has already been run.");
+    return;
+  }
+  // This captures whether a user has added manual build steps for Go
+  if (doesGoExtractionOutputExist(config)) {
+    // This log line is info level while Go extraction reconciliation is in beta.
+    // We will make it debug level once Go extraction reconciliation is GA.
+    logger.info(
+      "Won't run Go autobuild since at least one file of Go code has already been extracted."
+    );
+    return;
+  }
+  await runAutobuild(Language.go, config, logger);
+}
+
 async function run() {
   const startedAt = new Date();
   let uploadResult: UploadResult | undefined = undefined;
@@ -166,6 +244,8 @@ async function run() {
       logger
     );
 
+    await runAutobuildIfLegacyGoWorkflow(config, featureFlags, logger);
+
     dbCreationTimings = await runFinalize(
       outputDir,
       threads,

src/autobuild-action.ts

@@ -81,6 +81,9 @@ async function run() {
         process.chdir(workingDirectory);
       }
       await runAutobuild(language, config, logger);
+      if (language === Language.go) {
+        core.exportVariable("CODEQL_ACTION_DID_AUTOBUILD_GOLANG", "true");
+      }
     }
   } catch (error) {
     core.setFailed(

src/feature-flags.ts

@@ -12,6 +12,7 @@ export enum FeatureFlag {
   LuaTracerConfigEnabled = "lua_tracer_config_enabled",
   MlPoweredQueriesEnabled = "ml_powered_queries_enabled",
   TrapCachingEnabled = "trap_caching_enabled",
+  GolangExtractionReconciliationEnabled = "golang_extraction_reconciliation_enabled",
 }
 
 /**