new-log-viewer: Add log query support in StateContextProvider. (#80)

Author: Henry8192

new-log-viewer/src/contexts/StateContextProvider.tsx

@@ -21,6 +21,7 @@ import {
     EVENT_POSITION_ON_PAGE,
     FileSrcType,
     MainWorkerRespMessage,
+    QueryResults,
     WORKER_REQ_CODE,
     WORKER_RESP_CODE,
     WorkerReq,
@@ -57,11 +58,13 @@ interface StateContextType {
     numPages: number,
     onDiskFileSizeInBytes: number,
     pageNum: number,
+    queryResults: QueryResults,
 
     exportLogs: () => void,
     loadFile: (fileSrc: FileSrcType, cursor: CursorType) => void,
     loadPageByAction: (navAction: NavigationAction) => void,
     setLogLevelFilter: (newLogLevelFilter: LogLevelFilter) => void,
+    startQuery: (queryString: string, isRegex: boolean, isCaseSensitive: boolean) => void,
 }
 const StateContext = createContext<StateContextType>({} as StateContextType);
 
@@ -77,11 +80,13 @@ const STATE_DEFAULT: Readonly<StateContextType> = Object.freeze({
     numPages: 0,
     onDiskFileSizeInBytes: 0,
     pageNum: 0,
+    queryResults: new Map(),
 
     exportLogs: () => null,
     loadFile: () => null,
     loadPageByAction: () => null,
     setLogLevelFilter: () => null,
+    startQuery: () => null,
 });
 
 interface StateContextProviderProps {
@@ -229,17 +234,18 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
     const {filePath, logEventNum} = useContext(UrlContext);
 
     // States
+    const [exportProgress, setExportProgress] =
+        useState<Nullable<number>>(STATE_DEFAULT.exportProgress);
     const [fileName, setFileName] = useState<string>(STATE_DEFAULT.fileName);
     const [logData, setLogData] = useState<string>(STATE_DEFAULT.logData);
     const [numEvents, setNumEvents] = useState<number>(STATE_DEFAULT.numEvents);
     const [numPages, setNumPages] = useState<number>(STATE_DEFAULT.numPages);
     const [onDiskFileSizeInBytes, setOnDiskFileSizeInBytes] =
         useState(STATE_DEFAULT.onDiskFileSizeInBytes);
     const [pageNum, setPageNum] = useState<number>(STATE_DEFAULT.pageNum);
+    const [queryResults, setQueryResults] = useState<QueryResults>(STATE_DEFAULT.queryResults);
     const beginLineNumToLogEventNumRef =
         useRef<BeginLineNumToLogEventNumMap>(STATE_DEFAULT.beginLineNumToLogEventNum);
-    const [exportProgress, setExportProgress] =
-        useState<Nullable<number>>(STATE_DEFAULT.exportProgress);
 
     // Refs
     const logEventNumRef = useRef(logEventNum);
@@ -281,12 +287,42 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
                 });
                 break;
             }
+            case WORKER_RESP_CODE.QUERY_RESULT:
+                setQueryResults((v) => {
+                    args.results.forEach((resultsPerPage, queryPageNum) => {
+                        if (false === v.has(queryPageNum)) {
+                            v.set(queryPageNum, []);
+                        }
+                        v.get(queryPageNum)?.push(...resultsPerPage);
+                    });
+
+                    return v;
+                });
+                break;
             default:
                 console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`);
                 break;
         }
     }, [postPopUp]);
 
+    const startQuery = useCallback((
+        queryString: string,
+        isRegex: boolean,
+        isCaseSensitive: boolean
+    ) => {
+        setQueryResults(STATE_DEFAULT.queryResults);
+        if (null === mainWorkerRef.current) {
+            console.error("Unexpected null mainWorkerRef.current");
+
+            return;
+        }
+        workerPostReq(mainWorkerRef.current, WORKER_REQ_CODE.START_QUERY, {
+            queryString: queryString,
+            isRegex: isRegex,
+            isCaseSensitive: isCaseSensitive,
+        });
+    }, []);
+
     const exportLogs = useCallback(() => {
         if (null === mainWorkerRef.current) {
             console.error("Unexpected null mainWorkerRef.current");
@@ -442,11 +478,13 @@ const StateContextProvider = ({children}: StateContextProviderProps) => {
                 numPages: numPages,
                 onDiskFileSizeInBytes: onDiskFileSizeInBytes,
                 pageNum: pageNum,
+                queryResults: queryResults,
 
                 exportLogs: exportLogs,
                 loadFile: loadFile,
                 loadPageByAction: loadPageByAction,
                 setLogLevelFilter: setLogLevelFilter,
+                startQuery: startQuery,
             }}
         >
             {children}

new-log-viewer/src/services/LogFileManager/index.ts

@@ -1,3 +1,4 @@
+/* eslint max-lines: ["error", 400] */
 import {
     Decoder,
     DecoderOptionsType,
@@ -11,11 +12,16 @@ import {
     CursorType,
     EMPTY_PAGE_RESP,
     FileSrcType,
+    QueryResults,
     WORKER_RESP_CODE,
     WorkerResp,
 } from "../../typings/worker";
-import {EXPORT_LOGS_CHUNK_SIZE} from "../../utils/config";
+import {
+    EXPORT_LOGS_CHUNK_SIZE,
+    QUERY_CHUNK_SIZE,
+} from "../../utils/config";
 import {getChunkNum} from "../../utils/math";
+import {defer} from "../../utils/time";
 import {formatSizeInBytes} from "../../utils/units";
 import ClpIrDecoder from "../decoders/ClpIrDecoder";
 import JsonlDecoder from "../decoders/JsonlDecoder";
@@ -31,35 +37,43 @@ import {
  * Class to manage the retrieval and decoding of a given log file.
  */
 class LogFileManager {
+    readonly #fileName: string;
+
+    readonly #numEvents: number = 0;
+
     readonly #pageSize: number;
 
-    readonly #fileName: string;
+    #queryId: number = 0;
 
     readonly #onDiskFileSizeInBytes: number;
 
-    #decoder: Decoder;
+    readonly #onQueryResults: (queryResults: QueryResults) => void;
 
-    #numEvents: number = 0;
+    #decoder: Decoder;
 
     /**
      * Private constructor for LogFileManager. This is not intended to be invoked publicly.
      * Instead, use LogFileManager.create() to create a new instance of the class.
      *
-     * @param decoder
-     * @param fileName
-     * @param onDiskFileSizeInBytes
-     * @param pageSize Page size for setting up pagination.
+     * @param params
+     * @param params.decoder
+     * @param params.fileName
+     * @param params.onDiskFileSizeInBytes
+     * @param params.pageSize Page size for setting up pagination.
+     * @param params.onQueryResults
      */
-    constructor (
+    constructor ({decoder, fileName, onDiskFileSizeInBytes, pageSize, onQueryResults}: {
         decoder: Decoder,
         fileName: string,
         onDiskFileSizeInBytes: number,
         pageSize: number,
-    ) {
+        onQueryResults: (queryResults: QueryResults) => void,
+    }) {
+        this.#decoder = decoder;
         this.#fileName = fileName;
-        this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes;
         this.#pageSize = pageSize;
-        this.#decoder = decoder;
+        this.#onDiskFileSizeInBytes = onDiskFileSizeInBytes;
+        this.#onQueryResults = onQueryResults;
 
         // Build index for the entire file.
         const buildResult = decoder.build();
@@ -90,17 +104,26 @@ class LogFileManager {
      * File object.
      * @param pageSize Page size for setting up pagination.
      * @param decoderOptions Initial decoder options.
+     * @param onQueryResults
      * @return A Promise that resolves to the created LogFileManager instance.
      */
     static async create (
         fileSrc: FileSrcType,
         pageSize: number,
-        decoderOptions: DecoderOptionsType
+        decoderOptions: DecoderOptionsType,
+        onQueryResults: (queryResults: QueryResults) => void,
     ): Promise<LogFileManager> {
         const {fileName, fileData} = await loadFile(fileSrc);
         const decoder = await LogFileManager.#initDecoder(fileName, fileData, decoderOptions);
 
-        return new LogFileManager(decoder, fileName, fileData.length, pageSize);
+        return new LogFileManager({
+            decoder: decoder,
+            fileName: fileName,
+            onDiskFileSizeInBytes: fileData.length,
+            pageSize: pageSize,
+
+            onQueryResults: onQueryResults,
+        });
     }
 
     /**
@@ -254,6 +277,86 @@ class LogFileManager {
         };
     }
 
+    /**
+     * Creates a RegExp object based on the given query string and options, and starts querying the
+     * first log chunk.
+     *
+     * @param queryString
+     * @param isRegex
+     * @param isCaseSensitive
+     */
+    startQuery (queryString: string, isRegex: boolean, isCaseSensitive: boolean): void {
+        this.#queryId++;
+
+        // If the query string is empty, or there are no logs, return
+        if ("" === queryString || 0 === this.#numEvents) {
+            return;
+        }
+
+        // Construct query RegExp
+        const regexPattern = isRegex ?
+            queryString :
+            queryString.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+        const regexFlags = isCaseSensitive ?
+            "" :
+            "i";
+        const queryRegex = new RegExp(regexPattern, regexFlags);
+
+        this.#queryChunkAndScheduleNext(this.#queryId, 0, queryRegex);
+    }
+
+    /**
+     * Queries a chunk of log events, sends the results, and schedules the next chunk query if more
+     * log events remain.
+     *
+     * @param queryId
+     * @param chunkBeginIdx
+     * @param queryRegex
+     */
+    #queryChunkAndScheduleNext (
+        queryId: number,
+        chunkBeginIdx: number,
+        queryRegex: RegExp
+    ): void {
+        if (queryId !== this.#queryId) {
+            // Current task no longer corresponds to the latest query in the LogFileManager.
+            return;
+        }
+        const chunkEndIdx = Math.min(chunkBeginIdx + QUERY_CHUNK_SIZE, this.#numEvents);
+        const results: QueryResults = new Map();
+        const decodedEvents = this.#decoder.decodeRange(
+            chunkBeginIdx,
+            chunkEndIdx,
+            null !== this.#decoder.getFilteredLogEventMap()
+        );
+
+        decodedEvents?.forEach(([message, , , logEventNum]) => {
+            const matchResult = message.match(queryRegex);
+            if (null !== matchResult && "number" === typeof matchResult.index) {
+                const pageNum = Math.ceil(logEventNum / this.#pageSize);
+                if (false === results.has(pageNum)) {
+                    results.set(pageNum, []);
+                }
+                results.get(pageNum)?.push({
+                    logEventNum: logEventNum,
+                    message: message,
+                    matchRange: [
+                        matchResult.index,
+                        (matchResult.index + matchResult[0].length),
+                    ],
+                });
+            }
+        });
+
+        this.#onQueryResults(results);
+
+        if (chunkEndIdx < this.#numEvents) {
+            defer(() => {
+                this.#queryChunkAndScheduleNext(queryId, chunkEndIdx, queryRegex);
+            });
+        }
+    }
+
     /**
      * Gets the data that corresponds to the cursor.
      *

new-log-viewer/src/services/MainWorker.ts

@@ -5,6 +5,7 @@ import dayjsUtc from "dayjs/plugin/utc";
 import {LOG_LEVEL} from "../typings/logs";
 import {
     MainWorkerReqMessage,
+    QueryResults,
     WORKER_REQ_CODE,
     WORKER_RESP_CODE,
     WorkerResp,
@@ -36,6 +37,16 @@ const postResp = <T extends WORKER_RESP_CODE>(
     postMessage({code, args});
 };
 
+
+/**
+ * Post a response for a chunk of query results.
+ *
+ * @param queryResults
+ */
+const onQueryResults = (queryResults: QueryResults) => {
+    postResp(WORKER_RESP_CODE.QUERY_RESULT, {results: queryResults});
+};
+
 // eslint-disable-next-line no-warning-comments
 // TODO: Break this function up into smaller functions.
 // eslint-disable-next-line max-lines-per-function,max-statements
@@ -63,7 +74,8 @@ onmessage = async (ev: MessageEvent<MainWorkerReqMessage>) => {
                 LOG_FILE_MANAGER = await LogFileManager.create(
                     args.fileSrc,
                     args.pageSize,
-                    args.decoderOptions
+                    args.decoderOptions,
+                    onQueryResults
                 );
 
                 postResp(WORKER_RESP_CODE.LOG_FILE_INFO, {
@@ -97,6 +109,23 @@ onmessage = async (ev: MessageEvent<MainWorkerReqMessage>) => {
                     LOG_FILE_MANAGER.loadPage(args.cursor)
                 );
                 break;
+            case WORKER_REQ_CODE.START_QUERY:
+                if (null === LOG_FILE_MANAGER) {
+                    throw new Error("Log file manager hasn't been initialized");
+                }
+                if (
+                    "string" !== typeof args.queryString ||
+                    "boolean" !== typeof args.isRegex ||
+                    "boolean" !== typeof args.isCaseSensitive
+                ) {
+                    throw new Error("Invalid arguments for QUERY_LOG");
+                }
+                LOG_FILE_MANAGER.startQuery(
+                    args.queryString,
+                    args.isRegex,
+                    args.isCaseSensitive
+                );
+                break;
             default:
                 console.error(`Unexpected ev.data: ${JSON.stringify(ev.data)}`);
                 break;