Merge branch 'main' into ghae-endpoint

lib/runner.js

@@ -82,8 +82,8 @@ program
     .description("Initializes CodeQL")
     .requiredOption("--repository <repository>", "Repository name. (Required)")
     .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-    .requiredOption("--github-auth <auth>", "GitHub Apps token or personal access token. (Required)")
+    .option("--github-auth <auth>", "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead.")
-    .option("--external-repository-token <token>", "A token for fetching external config files and queries if they reside in a private repository.")
+    .option("--github-auth-stdin", "Read GitHub Apps token or personal access token from stdin.")
     .option("--languages <languages>", "Comma-separated list of languages to analyze. Otherwise detects and analyzes all supported languages from the repo.")
     .option("--queries <queries>", "Comma-separated list of additional queries to run. This overrides the same setting in a configuration file.")
     .option("--config-file <file>", "Path to config file.")
@@ -104,9 +104,9 @@ program
         logger.info(`Cleaning temp directory ${tempDir}`);
         fs.rmdirSync(tempDir, { recursive: true });
         fs.mkdirSync(tempDir, { recursive: true });
+        const auth = await util_1.getGitHubAuth(logger, cmd.githubAuth, cmd.githubAuthStdin);
         const apiDetails = {
-            auth: cmd.githubAuth,
+            auth,
-            externalRepoAuth: cmd.externalRepositoryToken,
             url: util_1.parseGithubUrl(cmd.githubUrl),
         };
         const gitHubVersion = await util_1.getGitHubVersion(apiDetails);
@@ -209,7 +209,8 @@ program
     .requiredOption("--commit <commit>", "SHA of commit that was analyzed. (Required)")
     .requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
     .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-    .requiredOption("--github-auth <auth>", "GitHub Apps token or personal access token. (Required)")
+    .option("--github-auth <auth>", "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead.")
+    .option("--github-auth-stdin", "Read GitHub Apps token or personal access token from stdin.")
     .option("--checkout-path <path>", "Checkout path. Default is the current working directory.")
     .option("--no-upload", "Do not upload results after analysis.")
     .option("--output-dir <dir>", "Directory to output SARIF files to. Default is in the temp directory.")
@@ -229,8 +230,9 @@ program
             throw new Error("Config file could not be found at expected location. " +
                 "Was the 'init' command run with the same '--temp-dir' argument as this command.");
         }
+        const auth = await util_1.getGitHubAuth(logger, cmd.githubAuth, cmd.githubAuthStdin);
         const apiDetails = {
-            auth: cmd.githubAuth,
+            auth,
             url: util_1.parseGithubUrl(cmd.githubUrl),
         };
         await analyze_1.runAnalyze(outputDir, util_1.getMemoryFlag(cmd.ram), util_1.getAddSnippetsFlag(cmd.addSnippets), util_1.getThreadsFlag(cmd.threads, logger), config, logger);
@@ -254,13 +256,15 @@ program
     .requiredOption("--commit <commit>", "SHA of commit that was analyzed. (Required)")
     .requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
     .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-    .requiredOption("--github-auth <auth>", "GitHub Apps token or personal access token. (Required)")
+    .option("--github-auth <auth>", "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead.")
+    .option("--github-auth-stdin", "Read GitHub Apps token or personal access token from stdin.")
     .option("--checkout-path <path>", "Checkout path. Default is the current working directory.")
     .option("--debug", "Print more verbose output", false)
     .action(async (cmd) => {
     const logger = logging_1.getRunnerLogger(cmd.debug);
+    const auth = await util_1.getGitHubAuth(logger, cmd.githubAuth, cmd.githubAuthStdin);
     const apiDetails = {
-        auth: cmd.githubAuth,
+        auth,
         url: util_1.parseGithubUrl(cmd.githubUrl),
     };
     try {

lib/util.js

@@ -257,4 +257,55 @@ function apiVersionInRange(version, minimumVersion, maximumVersion) {
     return undefined;
 }
 exports.apiVersionInRange = apiVersionInRange;
+/**
+ * Retrieves the github auth token for use with the runner. There are
+ * three possible locations for the token:
+ *
+ * 1. from the cli (considered insecure)
+ * 2. from stdin
+ * 3. from the GITHUB_TOKEN environment variable
+ *
+ * If both 1 & 2 are specified, then an error is thrown.
+ * If 1 & 3 or 2 & 3 are specified, then the environment variable is ignored.
+ *
+ * @param githubAuth a github app token or PAT
+ * @param fromStdIn read the github app token or PAT from stdin up to, but excluding the first whitespace
+ * @param readable the readable stream to use for getting the token (defaults to stdin)
+ *
+ * @return a promise resolving to the auth token.
+ */
+async function getGitHubAuth(logger, githubAuth, fromStdIn, readable = process.stdin) {
+    if (githubAuth && fromStdIn) {
+        throw new Error("Cannot specify both `--github-auth` and `--github-auth-stdin`. Please use `--github-auth-stdin`, which is more secure.");
+    }
+    if (githubAuth) {
+        logger.warning("Using `--github-auth` via the CLI is insecure. Use `--github-auth-stdin` instead.");
+        return githubAuth;
+    }
+    if (fromStdIn) {
+        return new Promise((resolve, reject) => {
+            let token = "";
+            readable.on("data", (data) => {
+                token += data.toString("utf8");
+            });
+            readable.on("end", () => {
+                token = token.split(/\s+/)[0].trim();
+                if (token) {
+                    resolve(token);
+                }
+                else {
+                    reject(new Error("Standard input is empty"));
+                }
+            });
+            readable.on("error", (err) => {
+                reject(err);
+            });
+        });
+    }
+    if (process.env.GITHUB_TOKEN) {
+        return process.env.GITHUB_TOKEN;
+    }
+    throw new Error("No GitHub authentication token was specified. Please provide a token via the GITHUB_TOKEN environment variable, or by adding the `--github-auth-stdin` flag and passing the token via standard input.");
+}
+exports.getGitHubAuth = getGitHubAuth;
 //# sourceMappingURL=util.js.map

lib/util.test.js

@@ -12,6 +12,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 const fs = __importStar(require("fs"));
 const os = __importStar(require("os"));
+const stream = __importStar(require("stream"));
 const github = __importStar(require("@actions/github"));
 const ava_1 = __importDefault(require("ava"));
 const sinon_1 = __importDefault(require("sinon"));
@@ -172,4 +173,34 @@ ava_1.default("getGitHubVersion", async (t) => {
     });
     t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
 });
+ava_1.default("getGitHubAuth", async (t) => {
+    const msgs = [];
+    const mockLogger = {
+        warning: (msg) => msgs.push(msg),
+    };
+    // eslint-disable-next-line @typescript-eslint/no-floating-promises
+    t.throwsAsync(async () => util.getGitHubAuth(mockLogger, "abc", true));
+    process.env.GITHUB_TOKEN = "123";
+    t.is("123", await util.getGitHubAuth(mockLogger, undefined, undefined));
+    t.is(msgs.length, 0);
+    t.is("abc", await util.getGitHubAuth(mockLogger, "abc", undefined));
+    t.is(msgs.length, 1); // warning expected
+    msgs.length = 0;
+    await mockStdInForAuth(t, mockLogger, "def", "def");
+    await mockStdInForAuth(t, mockLogger, "def", "", "def");
+    await mockStdInForAuth(t, mockLogger, "def", "def\n some extra garbage", "ghi");
+    await mockStdInForAuth(t, mockLogger, "defghi", "def", "ghi\n123");
+    await mockStdInForAuthExpectError(t, mockLogger, "");
+    await mockStdInForAuthExpectError(t, mockLogger, "", " ", "abc");
+    await mockStdInForAuthExpectError(t, mockLogger, "  def\n some extra garbage", "ghi");
+    t.is(msgs.length, 0);
+});
+async function mockStdInForAuth(t, mockLogger, expected, ...text) {
+    const stdin = stream.Readable.from(text);
+    t.is(expected, await util.getGitHubAuth(mockLogger, undefined, true, stdin));
+}
+async function mockStdInForAuthExpectError(t, mockLogger, ...text) {
+    const stdin = stream.Readable.from(text);
+    await t.throwsAsync(async () => util.getGitHubAuth(mockLogger, undefined, true, stdin));
+}
 //# sourceMappingURL=util.test.js.map

src/api-client.ts

@@ -20,7 +20,7 @@ export interface GitHubApiDetails {
 }
 
 export interface GitHubApiExternalRepoDetails {
-  externalRepoAuth: string | undefined;
+  externalRepoAuth?: string;
   url: string;
 }
 

src/runner.ts

@@ -20,6 +20,7 @@ import {
   getMemoryFlag,
   getThreadsFlag,
   parseGithubUrl,
+  getGitHubAuth,
 } from "./util";
 
 const program = new Command();
@@ -96,7 +97,7 @@ interface InitArgs {
   repository: string;
   githubUrl: string;
   githubAuth: string;
-  externalRepositoryToken: string | undefined;
+  githubAuthStdin: boolean;
   debug: boolean;
 }
 
@@ -105,13 +106,13 @@ program
   .description("Initializes CodeQL")
   .requiredOption("--repository <repository>", "Repository name. (Required)")
   .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-  .requiredOption(
+  .option(
     "--github-auth <auth>",
-    "GitHub Apps token or personal access token. (Required)"
+    "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
   )
   .option(
-    "--external-repository-token <token>",
+    "--github-auth-stdin",
-    "A token for fetching external config files and queries if they reside in a private repository."
+    "Read GitHub Apps token or personal access token from stdin."
   )
   .option(
     "--languages <languages>",
@@ -153,9 +154,14 @@ program
       fs.rmdirSync(tempDir, { recursive: true });
       fs.mkdirSync(tempDir, { recursive: true });
 
+      const auth = await getGitHubAuth(
+        logger,
+        cmd.githubAuth,
+        cmd.githubAuthStdin
+      );
+
       const apiDetails = {
-        auth: cmd.githubAuth,
+        auth,
-        externalRepoAuth: cmd.externalRepositoryToken,
         url: parseGithubUrl(cmd.githubUrl),
       };
 
@@ -315,6 +321,7 @@ interface AnalyzeArgs {
   ref: string;
   githubUrl: string;
   githubAuth: string;
+  githubAuthStdin: boolean;
   checkoutPath: string | undefined;
   upload: boolean;
   outputDir: string | undefined;
@@ -335,9 +342,13 @@ program
   )
   .requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
   .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-  .requiredOption(
+  .option(
     "--github-auth <auth>",
-    "GitHub Apps token or personal access token. (Required)"
+    "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
+  )
+  .option(
+    "--github-auth-stdin",
+    "Read GitHub Apps token or personal access token from stdin."
   )
   .option(
     "--checkout-path <path>",
@@ -379,8 +390,14 @@ program
         );
       }
 
+      const auth = await getGitHubAuth(
+        logger,
+        cmd.githubAuth,
+        cmd.githubAuthStdin
+      );
+
       const apiDetails = {
-        auth: cmd.githubAuth,
+        auth,
         url: parseGithubUrl(cmd.githubUrl),
       };
 
@@ -421,6 +438,7 @@ interface UploadArgs {
   commit: string;
   ref: string;
   githubUrl: string;
+  githubAuthStdin: boolean;
   githubAuth: string;
   checkoutPath: string | undefined;
   debug: boolean;
@@ -442,9 +460,13 @@ program
   )
   .requiredOption("--ref <ref>", "Name of ref that was analyzed. (Required)")
   .requiredOption("--github-url <url>", "URL of GitHub instance. (Required)")
-  .requiredOption(
+  .option(
     "--github-auth <auth>",
-    "GitHub Apps token or personal access token. (Required)"
+    "GitHub Apps token or personal access token. This option is insecure and deprecated, please use `--github-auth-stdin` instead."
+  )
+  .option(
+    "--github-auth-stdin",
+    "Read GitHub Apps token or personal access token from stdin."
   )
   .option(
     "--checkout-path <path>",
@@ -453,8 +475,13 @@ program
   .option("--debug", "Print more verbose output", false)
   .action(async (cmd: UploadArgs) => {
     const logger = getRunnerLogger(cmd.debug);
+    const auth = await getGitHubAuth(
+      logger,
+      cmd.githubAuth,
+      cmd.githubAuthStdin
+    );
     const apiDetails = {
-      auth: cmd.githubAuth,
+      auth,
       url: parseGithubUrl(cmd.githubUrl),
     };
     try {

src/util.test.ts

@@ -1,12 +1,13 @@
 import * as fs from "fs";
 import * as os from "os";
+import * as stream from "stream";
 
 import * as github from "@actions/github";
-import test from "ava";
+import test, { ExecutionContext } from "ava";
 import sinon from "sinon";
 
 import * as api from "./api-client";
-import { getRunnerLogger } from "./logging";
+import { getRunnerLogger, Logger } from "./logging";
 import { setupTests } from "./testing-utils";
 import * as util from "./util";
 
@@ -244,3 +245,62 @@ test("getGitHubVersion", async (t) => {
   });
   t.deepEqual({ type: util.GitHubVariant.DOTCOM }, v3);
 });
+
+test("getGitHubAuth", async (t) => {
+  const msgs: string[] = [];
+  const mockLogger = ({
+    warning: (msg: string) => msgs.push(msg),
+  } as unknown) as Logger;
+
+  // eslint-disable-next-line @typescript-eslint/no-floating-promises
+  t.throwsAsync(async () => util.getGitHubAuth(mockLogger, "abc", true));
+
+  process.env.GITHUB_TOKEN = "123";
+  t.is("123", await util.getGitHubAuth(mockLogger, undefined, undefined));
+  t.is(msgs.length, 0);
+  t.is("abc", await util.getGitHubAuth(mockLogger, "abc", undefined));
+  t.is(msgs.length, 1); // warning expected
+
+  msgs.length = 0;
+  await mockStdInForAuth(t, mockLogger, "def", "def");
+  await mockStdInForAuth(t, mockLogger, "def", "", "def");
+  await mockStdInForAuth(
+    t,
+    mockLogger,
+    "def",
+    "def\n some extra garbage",
+    "ghi"
+  );
+  await mockStdInForAuth(t, mockLogger, "defghi", "def", "ghi\n123");
+
+  await mockStdInForAuthExpectError(t, mockLogger, "");
+  await mockStdInForAuthExpectError(t, mockLogger, "", " ", "abc");
+  await mockStdInForAuthExpectError(
+    t,
+    mockLogger,
+    "  def\n some extra garbage",
+    "ghi"
+  );
+  t.is(msgs.length, 0);
+});
+
+async function mockStdInForAuth(
+  t: ExecutionContext<any>,
+  mockLogger: Logger,
+  expected: string,
+  ...text: string[]
+) {
+  const stdin = stream.Readable.from(text) as any;
+  t.is(expected, await util.getGitHubAuth(mockLogger, undefined, true, stdin));
+}
+
+async function mockStdInForAuthExpectError(
+  t: ExecutionContext<unknown>,
+  mockLogger: Logger,
+  ...text: string[]
+) {
+  const stdin = stream.Readable.from(text) as any;
+  await t.throwsAsync(async () =>
+    util.getGitHubAuth(mockLogger, undefined, true, stdin)
+  );
+}

src/util.ts

@@ -1,6 +1,7 @@
 import * as fs from "fs";
 import * as os from "os";
 import * as path from "path";
+import { Readable } from "stream";
 
 import * as core from "@actions/core";
 import * as semver from "semver";
@@ -311,3 +312,68 @@ export function apiVersionInRange(
   }
   return undefined;
 }
+
+/**
+ * Retrieves the github auth token for use with the runner. There are
+ * three possible locations for the token:
+ *
+ * 1. from the cli (considered insecure)
+ * 2. from stdin
+ * 3. from the GITHUB_TOKEN environment variable
+ *
+ * If both 1 & 2 are specified, then an error is thrown.
+ * If 1 & 3 or 2 & 3 are specified, then the environment variable is ignored.
+ *
+ * @param githubAuth a github app token or PAT
+ * @param fromStdIn read the github app token or PAT from stdin up to, but excluding the first whitespace
+ * @param readable the readable stream to use for getting the token (defaults to stdin)
+ *
+ * @return a promise resolving to the auth token.
+ */
+export async function getGitHubAuth(
+  logger: Logger,
+  githubAuth: string | undefined,
+  fromStdIn: boolean | undefined,
+  readable = process.stdin as Readable
+): Promise<string> {
+  if (githubAuth && fromStdIn) {
+    throw new Error(
+      "Cannot specify both `--github-auth` and `--github-auth-stdin`. Please use `--github-auth-stdin`, which is more secure."
+    );
+  }
+
+  if (githubAuth) {
+    logger.warning(
+      "Using `--github-auth` via the CLI is insecure. Use `--github-auth-stdin` instead."
+    );
+    return githubAuth;
+  }
+
+  if (fromStdIn) {
+    return new Promise((resolve, reject) => {
+      let token = "";
+      readable.on("data", (data) => {
+        token += data.toString("utf8");
+      });
+      readable.on("end", () => {
+        token = token.split(/\s+/)[0].trim();
+        if (token) {
+          resolve(token);
+        } else {
+          reject(new Error("Standard input is empty"));
+        }
+      });
+      readable.on("error", (err) => {
+        reject(err);
+      });
+    });
+  }
+
+  if (process.env.GITHUB_TOKEN) {
+    return process.env.GITHUB_TOKEN;
+  }
+
+  throw new Error(
+    "No GitHub authentication token was specified. Please provide a token via the GITHUB_TOKEN environment variable, or by adding the `--github-auth-stdin` flag and passing the token via standard input."
+  );
+}