chore: try to speed up integration test downloads (#6611)

Author: Jason3S

integration-tests/package.json

@@ -29,6 +29,7 @@
   "dependencies": {
     "@cspell/cspell-types": "workspace:*",
     "@octokit/rest": "^21.0.2",
+    "ansi-escapes": "^7.0.0",
     "chalk": "^5.3.0",
     "commander": "^12.1.0",
     "csv-parse": "^5.6.0",

integration-tests/src/PrefixLogger.ts

@@ -1,11 +1,15 @@
 /* eslint-disable @typescript-eslint/no-explicit-any */
 
-import { errorWithPrefix, logWithPrefix } from './outputHelper.js';
+import { errorWithPrefix, logWithPrefix, outputWithPrefix } from './outputHelper.js';
 import type { Logger } from './types.js';
 
 export class PrefixLogger implements Logger {
     constructor(readonly prefix: string) {}
 
+    readonly output = (message: string, ...optionalParams: unknown[]): void => {
+        outputWithPrefix(this.prefix, message, ...optionalParams);
+    };
+
     readonly log = (message?: any, ...optionalParams: any[]): void => {
         logWithPrefix(this.prefix, message, ...optionalParams);
     };

integration-tests/src/outputHelper.ts

@@ -41,15 +41,44 @@ export function prefix(pfx: string, text: string): string {
     return text
         .split('\n')
         .map((line) => pfx + line)
-        .join('\n');
+        .join('\n')
+        .replaceAll('\r\n', '\n')
+        .replaceAll('\r', '\r' + pfx);
+}
+
+let lastPrefix = '';
+let pending = false;
+
+function clearLastPrefix() {
+    if (pending) {
+        process.stdout.write('\r\n');
+        pending = false;
+    }
+    lastPrefix = '';
+}
+
+function setLastPrefix(pfx: string) {
+    if (pending && lastPrefix !== pfx) {
+        process.stdout.write('\r\n');
+    }
+    pending = true;
+    lastPrefix = pfx;
+}
+
+export function outputWithPrefix(pfx: string, text: string, ...params: unknown[]): void {
+    setLastPrefix(pfx);
+    const s = text ? format(text, ...params) : '';
+    process.stdout.write(prefix(pfx, s).replaceAll('\n', '\r\n'));
 }
 
 export function logWithPrefix(pfx: string, text: string, ...params: unknown[]): void {
+    clearLastPrefix();
     const s = text ? format(text, ...params) : '';
     console.log(prefix(pfx, s));
 }
 
 export function errorWithPrefix(pfx: string, text: string, ...params: unknown[]): void {
+    clearLastPrefix();
     const s = text ? format(text, ...params) : '';
     console.error(prefix(pfx, red(s)));
 }

integration-tests/src/repositoryHelper.ts

@@ -3,6 +3,7 @@ import * as Path from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { Octokit } from '@octokit/rest';
+import { eraseEndLine } from 'ansi-escapes';
 import chalk from 'chalk';
 import { simpleGit } from 'simple-git';
 
@@ -85,7 +86,11 @@ export async function checkoutRepositoryAsync(
     if (!fs.existsSync(path)) {
         try {
             const repoInfo = await fetchRepositoryInfoForRepo(url);
-            const c = await cloneRepo(logger, url, path, !branch || branch === repoInfo.defaultBranch);
+            const cloneOptions: CloneOptions = {
+                commit: commit || branch || repoInfo.defaultBranch,
+                depth: 1,
+            };
+            const c = await cloneRepoFast(logger, url, path, cloneOptions);
             if (!c) {
                 return false;
             }
@@ -108,7 +113,7 @@ export async function checkoutRepositoryAsync(
     return true;
 }
 
-async function cloneRepo(
+async function _cloneRepo(
     { log, error }: Logger,
     url: string,
     path: string,
@@ -187,3 +192,54 @@ function getOctokit(auth?: string | undefined): Octokit {
     const options = auth ? { auth } : undefined;
     return new Octokit(options);
 }
+
+interface CloneOptions {
+    commit: string;
+    depth?: number;
+}
+
+/**
+ * Clone a repository
+ * ```sh
+ * mkdir <repo-name>
+ * cd <repo-name>
+ * git init
+ * git remote add origin <repository-url>
+ * git fetch origin <commit-hash> --depth=1
+ * git checkout FETCH_HEAD
+ * ```
+ * @param logger
+ * @param url
+ * @param path
+ * @param options
+ * @returns
+ */
+async function cloneRepoFast(logger: Logger, url: string, path: string, options: CloneOptions): Promise<boolean> {
+    const { log, error } = logger;
+    const stdOut = logger.output || log;
+    log(`Cloning ${url}`);
+    await mkdirp(path);
+    const gitOptions = [];
+    if (options.depth) {
+        gitOptions.push(`--depth=${options.depth}`);
+    }
+    try {
+        const git = simpleGit({
+            baseDir: path,
+            progress({ method, stage, progress }) {
+                stdOut(`\r${method} ${stage} stage ${progress}% complete${eraseEndLine}`);
+            },
+        });
+        log(`Init ${url}`);
+        await git.init().addRemote('origin', url);
+        log(`Fetch ${url}`);
+        await git.fetch('origin', options.commit, gitOptions);
+        log(`Checkout ${url}`);
+        await git.checkout('FETCH_HEAD');
+        log(`Cloned: ${url} with options: ${gitOptions.join(' ')}`);
+    } catch (e) {
+        error(e);
+        return false;
+    }
+    return true;
+}

integration-tests/src/types.ts

@@ -1,4 +1,5 @@
 export interface Logger {
+    output?: (message: string, ...optionalParams: unknown[]) => void;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     log: (message?: any, ...optionalParams: any[]) => void;
     // eslint-disable-next-line @typescript-eslint/no-explicit-any