fix(cspell-tools): support adding directives (#5860)

Author: Jason3S

packages/cspell-tools/cspell-tools.config.schema.json

@@ -144,6 +144,13 @@
           "description": "gzip the file?",
           "type": "boolean"
         },
+        "dictionaryDirectives": {
+          "description": "Injects `cspell-dictionary` directives into the dictionary header.\n\nExample:\n\n```ini # cspell-dictionary: no-generate-alternatives ```\n\nKnown Directives: ```yaml\n- split # Tell the dictionary loader to split words\n- no-split # Tell the dictionary loader to not split words (default)\n- generate-alternatives # Tell the dictionary loader to generate alternate spellings (default)\n- no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings ```",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
         "excludeWordsFrom": {
           "description": "Words from the sources that are found in `excludeWordsFrom` files will not be added to the dictionary.",
           "items": {
@@ -156,7 +163,7 @@
           "description": "Format of the dictionary."
         },
         "generateNonStrict": {
-          "default": true,
+          "default": false,
           "description": "Generate lower case / accent free versions of words.",
           "type": "boolean"
         },
@@ -165,7 +172,7 @@
           "type": "string"
         },
         "sort": {
-          "default": ": true",
+          "default": true,
           "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.",
           "type": "boolean"
         },
@@ -221,8 +228,15 @@
         "boolean"
       ]
     },
+    "dictionaryDirectives": {
+      "description": "Injects `cspell-dictionary` directives into the dictionary header.\n\nExample:\n\n```ini # cspell-dictionary: no-generate-alternatives ```\n\nKnown Directives: ```yaml\n- split # Tell the dictionary loader to split words\n- no-split # Tell the dictionary loader to not split words (default)\n- generate-alternatives # Tell the dictionary loader to generate alternate spellings (default)\n- no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings ```",
+      "items": {
+        "type": "string"
+      },
+      "type": "array"
+    },
     "generateNonStrict": {
-      "default": true,
+      "default": false,
       "description": "Generate lower case / accent free versions of words.",
       "type": "boolean"
     },
@@ -240,7 +254,7 @@
       "type": "string"
     },
     "sort": {
-      "default": ": true",
+      "default": true,
       "description": "Sort the words in the resulting dictionary. Does not apply to `trie` based formats.",
       "type": "boolean"
     },

packages/cspell-tools/src/compiler/CompileOptions.ts

@@ -16,4 +16,16 @@ export interface CompileOptions {
      * @returns `true` to keep the word, `false` to exclude it.
      */
     filter?: (word: string) => boolean;
+
+    /**
+     * Injects `cspell-dictionary` directives into the dictionary header.
+     *
+     * Example:
+     *
+     * ```ini
+     * # cspell-dictionary: no-generate-alternatives
+     * ```
+     *
+     */
+    dictionaryDirectives?: string[] | undefined;
 }

packages/cspell-tools/src/compiler/__snapshots__/wordListCompiler.test.ts.snap

@@ -1,5 +1,12 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`Validate the wordListCompiler > Specify directives 1`] = `
+{
+  "error": "",
+  "log": "",
+}
+`;
+
 exports[`Validate the wordListCompiler > a simple hunspell dictionary depth 0 1`] = `
 {
   "error": "",

packages/cspell-tools/src/compiler/compile.ts

@@ -53,14 +53,21 @@ export async function compile(request: CompileRequest, options?: CompileOptions)
     };
     const conditional = options?.conditionalBuild || false;
     const checksumFile = resolveChecksumFile(request.checksumFile || conditional, rootDir);
+    const dictionaryDirectives = request.dictionaryDirectives;
 
     const dependencies = new Set<string>();
 
     for (const target of targets) {
         const keep = options?.filter?.(target) ?? true;
         if (!keep) continue;
         const adjustedTarget: Target = { ...targetOptions, ...target };
-        const deps = await compileTarget(adjustedTarget, request, { rootDir, cwd, conditional, checksumFile });
+        const deps = await compileTarget(adjustedTarget, request, {
+            rootDir,
+            cwd,
+            conditional,
+            checksumFile,
+            dictionaryDirectives,
+        });
         deps.forEach((dep) => dependencies.add(dep));
     }
 
@@ -87,6 +94,7 @@ interface CompileTargetOptions {
     cwd: string | undefined;
     conditional: boolean;
     checksumFile: string | undefined;
+    dictionaryDirectives: string[] | undefined;
 }
 
 export async function compileTarget(
@@ -98,6 +106,7 @@ export async function compileTarget(
     const { rootDir, cwd, checksumFile, conditional } = compileOptions;
     const { format, sources, trieBase, sort = true, generateNonStrict = false, excludeWordsFrom } = target;
     const targetDirectory = path.resolve(rootDir, target.targetDirectory ?? cwd ?? process.cwd());
+    const dictionaryDirectives = compileOptions.dictionaryDirectives;
 
     const excludeFilter = await createExcludeFilter(excludeWordsFrom);
 
@@ -114,7 +123,12 @@ export async function compileTarget(
         opAwaitAsync(),
     );
     const filesToProcess: FileToProcess[] = await toArray(filesToProcessAsync);
-    const normalizer = normalizeTargetWords({ sort: useTrie || sort, generateNonStrict, filter: excludeFilter });
+    const normalizer = normalizeTargetWords({
+        sort: useTrie || sort,
+        generateNonStrict,
+        filter: excludeFilter,
+        dictionaryDirectives,
+    });
     const checksumRoot = (checksumFile && path.dirname(checksumFile)) || rootDir;
 
     const deps = [...calculateDependencies(filename, filesToProcess, excludeWordsFrom, checksumRoot)];
@@ -135,10 +149,11 @@ export async function compileTarget(
                   trie3: format === 'trie3',
                   trie4: format === 'trie4',
                   generateNonStrict: generateNonStrictTrie,
+                  dictionaryDirectives: undefined,
               });
           }
         : async (words: Iterable<string>, dst: string) => {
-              return compileWordList(pipe(words, normalizer), dst, { sort, generateNonStrict });
+              return compileWordList(pipe(words, normalizer), dst, { sort, generateNonStrict, dictionaryDirectives });
           };
 
     await processFiles(action, filesToProcess, filename);

packages/cspell-tools/src/compiler/wordListCompiler.test.ts

@@ -70,7 +70,7 @@ describe('Validate the wordListCompiler', () => {
         const output = await fsp.readFile(destName, 'utf8');
         expect(output).toBe(
             wordListHeader +
-                '\n' +
+                '\n\n' +
                 citiesSorted +
                 citiesSorted
                     .toLowerCase()
@@ -115,7 +115,7 @@ describe('Validate the wordListCompiler', () => {
         const destName = path.join(temp, 'example0.txt');
         await compileWordList(source, destName, compileOpt(false));
         const output = await fsp.readFile(destName, 'utf8');
-        expect(output).toBe(__testing__.wordListHeader + '\n' + 'hello\ntry\nwork\n');
+        expect(output).toBe(__testing__.wordListHeader + '\n\n' + 'hello\ntry\nwork\n');
         expect(consoleOutput()).toMatchSnapshot();
     });
 
@@ -144,6 +144,33 @@ describe('Validate the wordListCompiler', () => {
         );
         expect(consoleOutput()).toMatchSnapshot();
     });
+
+    test('Specify directives', async () => {
+        const source = await streamSourceWordsFromFile(path.join(samples, 'hunspell/example.dic'), {
+            ...readOptions,
+            maxDepth: 1,
+        });
+        const destName = path.join(temp, 'example2.txt');
+        await compileWordList(source, destName, compileOpt(false, false, ['no-generate-alternatives']));
+        const output = await fsp.readFile(destName, 'utf8');
+        expect(output.split('\n')).toEqual(
+            `\
+
+            # cspell-tools: keep-case no-split
+            # cspell-dictionary: no-generate-alternatives
+
+            hello
+            rework
+            tried
+            try
+            work
+            worked
+        `
+                .split('\n')
+                .map((line) => line.trim()),
+        );
+        expect(consoleOutput()).toMatchSnapshot();
+    });
 });
 
 describe('Validate Larger Dictionary', () => {
@@ -202,8 +229,8 @@ function legacyNormalizeWords(lines: Iterable<string>): Iterable<string> {
     );
 }
 
-function compileOpt(sort: boolean, generateNonStrict = true): CompileOptions {
-    return { sort, generateNonStrict };
+function compileOpt(sort: boolean, generateNonStrict = true, dictionaryDirectives?: string[]): CompileOptions {
+    return { sort, generateNonStrict, dictionaryDirectives };
 }
 
 // const cities = `\

packages/cspell-tools/src/compiler/wordListCompiler.ts

@@ -15,8 +15,7 @@ const mkdirp = async (p: string) => {
 
 // Indicate that a word list has already been processed.
 const wordListHeader = `
-# cspell-tools: keep-case no-split
-`;
+# cspell-tools: keep-case no-split`;
 const wordListHeaderLines = wordListHeader.split('\n').map((a) => a.trim());
 
 export async function compileWordList(
@@ -26,7 +25,10 @@ export async function compileWordList(
 ): Promise<void> {
     const finalLines = normalize(lines, options);
 
-    const finalSeq = pipe(wordListHeaderLines, opAppend(finalLines));
+    const directives = options.dictionaryDirectives ?? [];
+    const directivesLines = directives.map((a) => `# cspell-dictionary: ${a}`);
+
+    const finalSeq = pipe([...wordListHeaderLines, ...directivesLines, ''], opAppend(finalLines));
 
     return createWordListTarget(destFilename)(finalSeq);
 }

packages/cspell-tools/src/config/config.ts

@@ -45,14 +45,14 @@ export interface Experimental {
 export interface CompileTargetOptions {
     /**
      * Generate lower case / accent free versions of words.
-     * @default true
+     * @default false
      */
     generateNonStrict?: boolean | undefined;
 
     /**
      * Sort the words in the resulting dictionary.
      * Does not apply to `trie` based formats.
-     * @default: true
+     * @default true
      */
     sort?: boolean | undefined;
 
@@ -66,6 +66,25 @@ export interface CompileTargetOptions {
      * dictionary.
      */
     allowedSplitWords?: FilePath | FilePath[] | undefined;
+
+    /**
+     * Injects `cspell-dictionary` directives into the dictionary header.
+     *
+     * Example:
+     *
+     * ```ini
+     * # cspell-dictionary: no-generate-alternatives
+     * ```
+     *
+     * Known Directives:
+     * ```yaml
+     * - split # Tell the dictionary loader to split words
+     * - no-split # Tell the dictionary loader to not split words (default)
+     * - generate-alternatives # Tell the dictionary loader to generate alternate spellings (default)
+     * - no-generate-alternatives # Tell the dictionary loader to not generate alternate spellings
+     * ```
+     */
+    dictionaryDirectives?: string[] | undefined;
 }
 
 export interface Target extends CompileTargetOptions {