GH 677 First Pass: Glob-Style Wildcards (#697)

docs/sections/user_guide/cli/tools/config/validate-verbose.out

@@ -1,21 +1,21 @@
-[2025-01-15T20:22:34]    DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose
-[2025-01-15T20:22:34]    DEBUG [dereference] Dereferencing, current value:
-[2025-01-15T20:22:34]    DEBUG [dereference]   values:
-[2025-01-15T20:22:34]    DEBUG [dereference]     greeting: Hello
-[2025-01-15T20:22:34]    DEBUG [dereference]     recipient: World
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendering: values
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendered: values
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendering: greeting
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendered: greeting
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendering: Hello
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendered: Hello
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendering: recipient
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendered: recipient
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendering: World
-[2025-01-15T20:22:34]    DEBUG [dereference] Rendered: World
-[2025-01-15T20:22:34]    DEBUG [dereference] Dereferencing, final value:
-[2025-01-15T20:22:34]    DEBUG [dereference]   values:
-[2025-01-15T20:22:34]    DEBUG [dereference]     greeting: Hello
-[2025-01-15T20:22:34]    DEBUG [dereference]     recipient: World
-[2025-01-15T20:22:34]    DEBUG Using schema file: schema.jsonschema
-[2025-01-15T20:22:34]     INFO 0 schema-validation errors found in config
+[2025-02-15T07:02:49]    DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose
+[2025-02-15T07:02:49]    DEBUG [dereference] Dereferencing, current value:
+[2025-02-15T07:02:49]    DEBUG [dereference]   values:
+[2025-02-15T07:02:49]    DEBUG [dereference]     greeting: Hello
+[2025-02-15T07:02:49]    DEBUG [dereference]     recipient: World
+[2025-02-15T07:02:49]    DEBUG [dereference] Rendering: values
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendered: values
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendering: greeting
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendered: greeting
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendering: Hello
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendered: Hello
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendering: recipient
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendered: recipient
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendering: World
+[2025-02-15T07:02:50]    DEBUG [dereference] Rendered: World
+[2025-02-15T07:02:50]    DEBUG [dereference] Dereferencing, final value:
+[2025-02-15T07:02:50]    DEBUG [dereference]   values:
+[2025-02-15T07:02:50]    DEBUG [dereference]     greeting: Hello
+[2025-02-15T07:02:50]    DEBUG [dereference]     recipient: World
+[2025-02-15T07:02:50]    DEBUG Validating config against external schema file: schema.jsonschema
+[2025-02-15T07:02:50]     INFO 0 schema-validation errors found in config

docs/sections/user_guide/cli/tools/fs.rst

@@ -27,124 +27,91 @@ Source paths prefixed with ``http://`` or ``https://`` will be copied from their
 Examples
 ^^^^^^^^
 
-Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths
+Given a config containing a mapping from local-filesystem destination paths to source paths
 
-.. literalinclude:: fs/copy-config.yaml
+.. literalinclude:: fs/copy.yaml
    :language: yaml
-.. literalinclude:: fs/copy-exec.cmd
+.. literalinclude:: fs/copy.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/copy-exec.out
+.. literalinclude:: fs/copy.out
    :language: text
 
 Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source.
 
 The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example:
 
-.. literalinclude:: fs/copy-config-timedep.yaml
+.. literalinclude:: fs/copy-timedep.yaml
    :language: yaml
-.. literalinclude:: fs/copy-exec-timedep.cmd
+.. literalinclude:: fs/copy-timedep.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/copy-exec-timedep.out
+.. literalinclude:: fs/copy-timedep.out
    :language: text
 
 The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory:
 
-.. literalinclude:: fs/copy-config.yaml
+.. literalinclude:: fs/copy.yaml
    :language: yaml
-.. literalinclude:: fs/copy-exec-no-target-dir-err.cmd
+.. literalinclude:: fs/copy-no-target-dir-err.cmd
    :language: text
    :emphasize-lines: 1
-.. literalinclude:: fs/copy-exec-no-target-dir-err.out
+.. literalinclude:: fs/copy-no-target-dir-err.out
    :language: text
 
 When the ``--report`` option is specified, a report of files not copied ("not-ready") and copied ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files:
 
-.. literalinclude:: fs/copy-config-report.yaml
+.. literalinclude:: fs/copy-report.yaml
    :language: yaml
-.. literalinclude:: fs/copy-exec-report.cmd
+.. literalinclude:: fs/copy-report.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/copy-exec-report.out
+.. literalinclude:: fs/copy-report.out
    :language: text
 
 Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``:
 
-.. literalinclude:: fs/copy-exec-report-jq.cmd
+.. literalinclude:: fs/copy-report-jq.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/copy-exec-report-jq.out
+.. literalinclude:: fs/copy-report-jq.out
    :language: text
 
-``link``
---------
-
-The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block <files_yaml>`.
-
-.. literalinclude:: fs/link-help.cmd
-   :language: text
-   :emphasize-lines: 1
-.. literalinclude:: fs/link-help.out
-   :language: text
-
-Examples
-^^^^^^^^
-
-Given ``link-config.yaml`` containing
+Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern:
 
-.. literalinclude:: fs/link-config.yaml
+.. literalinclude:: fs/copy-glob.yaml
    :language: yaml
-.. literalinclude:: fs/link-exec.cmd
+.. literalinclude:: fs/copy-glob.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/link-exec.out
+.. literalinclude:: fs/copy-glob.out
    :language: text
 
-Here, ``foo`` and ``bar`` are symbolic links.
+The ``--report`` output can be especially useful in combination with wildcards to allow downstream logic to process a set of copied files whose identity is not known in advance.
 
-The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example:
+Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library :python:`shutil.copytree <shutil.html#shutil.copytree>`. For more control, including file-grained include and exclude, consider the unrivaled `rsync <https://rsync.samba.org/>`_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via :python:`subprocess <subprocess.html>` from Python.
 
-.. literalinclude:: fs/link-config-timedep.yaml
-   :language: yaml
-.. literalinclude:: fs/link-exec-timedep.cmd
-   :language: text
-   :emphasize-lines: 2
-.. literalinclude:: fs/link-exec-timedep.out
-   :language: text
-
-The ``--target-dir`` option need not be specified when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory:
-
-.. literalinclude:: fs/link-config.yaml
-   :language: yaml
-.. literalinclude:: fs/link-exec-no-target-dir-err.cmd
-   :language: text
-   :emphasize-lines: 1
-.. literalinclude:: fs/link-exec-no-target-dir-err.out
-   :language: text
+See :ref:`files_yaml` for more information on the semantics of the ``!glob`` tag and wildcard copies.
 
-When the ``--report`` option is specified, a report of files not linked ("not-ready") and linked ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files:
+``link``
+--------
 
-.. literalinclude:: fs/link-config-report.yaml
-   :language: yaml
-.. literalinclude:: fs/link-exec-report.cmd
-   :language: text
-   :emphasize-lines: 2
-.. literalinclude:: fs/link-exec-report.out
-   :language: text
+The ``link`` action stages items in a target directory by creating symbolic links to files, directories, or other symbolic links. It otherwise behaves similarly to ``copy`` (see above), but note the following:
 
-Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``:
+* In addition to file, directories and other symbolic links can be linked.
+* HTTP(S) sources are not supported.
+* Support for wildcard source values is the same as for ``link``.
 
-.. literalinclude:: fs/link-exec-report-jq.cmd
+.. literalinclude:: fs/link-help.cmd
    :language: text
-   :emphasize-lines: 2
-.. literalinclude:: fs/link-exec-report-jq.out
+   :emphasize-lines: 1
+.. literalinclude:: fs/link-help.out
    :language: text
 
 ``makedirs``
 ------------
 
-The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block <makedirs_yaml>`.
+The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block <makedirs_yaml>`, which must nest under a ``makedirs:`` key.
 
 .. literalinclude:: fs/makedirs-help.cmd
    :language: text
@@ -155,50 +122,34 @@ The ``makedirs`` action creates directories. Any ``KEY`` positional arguments ar
 Examples
 ^^^^^^^^
 
-Given ``makedirs-config.yaml`` containing
+Given a config containing
 
-.. literalinclude:: fs/makedirs-config.yaml
+.. literalinclude:: fs/makedirs.yaml
    :language: yaml
-.. literalinclude:: fs/makedirs-exec.cmd
+.. literalinclude:: fs/makedirs.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/makedirs-exec.out
+.. literalinclude:: fs/makedirs.out
    :language: text
 
 The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example:
 
-.. literalinclude:: fs/makedirs-config-timedep.yaml
+.. literalinclude:: fs/makedirs-timedep.yaml
    :language: yaml
-.. literalinclude:: fs/makedirs-exec-timedep.cmd
+.. literalinclude:: fs/makedirs-timedep.cmd
    :language: text
    :emphasize-lines: 2
-.. literalinclude:: fs/makedirs-exec-timedep.out
+.. literalinclude:: fs/makedirs-timedep.out
    :language: text
 
 The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory:
 
-.. literalinclude:: fs/makedirs-config.yaml
+.. literalinclude:: fs/makedirs.yaml
    :language: yaml
-.. literalinclude:: fs/makedirs-exec-no-target-dir-err.cmd
+.. literalinclude:: fs/makedirs-no-target-dir-err.cmd
    :language: text
    :emphasize-lines: 1
-.. literalinclude:: fs/makedirs-exec-no-target-dir-err.out
+.. literalinclude:: fs/makedirs-no-target-dir-err.out
    :language: text
 
-When the ``--report`` option is specified, a report of directories not created ("not-ready") and created ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files:
-
-.. literalinclude:: fs/makedirs-config-report.yaml
-   :language: yaml
-.. literalinclude:: fs/makedirs-exec-report.cmd
-   :language: text
-   :emphasize-lines: 5
-.. literalinclude:: fs/makedirs-exec-report.out
-   :language: text
-
-Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``:
-
-.. literalinclude:: fs/makedirs-exec-report-jq.cmd
-   :language: text
-   :emphasize-lines: 5
-.. literalinclude:: fs/makedirs-exec-report-jq.out
-   :language: text
+The ``--report`` option behaves the same as for ``link`` (see above).

docs/sections/user_guide/cli/tools/fs/.gitignore

@@ -1,7 +1 @@
-copy-dst
-copy-dst-report-jq
-copy-dst-timedep
-link-dst
-link-dst-report
-link-dst-report-jq
-link-dst-timedep
+dst/

docs/sections/user_guide/cli/tools/fs/copy-glob.cmd

@@ -0,0 +1,4 @@
+rm -rf dst/copy-glob
+uw fs copy --report --target-dir dst/copy-glob --config-file copy-glob.yaml
+echo
+tree dst/copy-glob

docs/sections/user_guide/cli/tools/fs/copy-glob.out

@@ -0,0 +1,24 @@
+[2025-02-15T18:52:47]     INFO Validating config against internal schema: files-to-stage
+[2025-02-15T18:52:47]     INFO 0 schema-validation errors found in fs config
+[2025-02-15T18:52:47]  WARNING Ignoring directory src/20240529
+[2025-02-15T18:52:47]     INFO File src/foo: Ready
+[2025-02-15T18:52:47]     INFO File src/bar: Ready
+[2025-02-15T18:52:47]     INFO Copy src/foo -> dst/copy-glob/dst/foo: Executing
+[2025-02-15T18:52:47]     INFO Copy src/foo -> dst/copy-glob/dst/foo: Ready
+[2025-02-15T18:52:47]     INFO Copy src/bar -> dst/copy-glob/dst/bar: Executing
+[2025-02-15T18:52:47]     INFO Copy src/bar -> dst/copy-glob/dst/bar: Ready
+[2025-02-15T18:52:47]     INFO File copies: Ready
+{
+  "not-ready": [],
+  "ready": [
+    "dst/copy-glob/dst/foo",
+    "dst/copy-glob/dst/bar"
+  ]
+}
+
+dst/copy-glob
+└── dst
+    ├── bar
+    └── foo
+
+2 directories, 2 files

docs/sections/user_guide/cli/tools/fs/copy-glob.yaml

@@ -0,0 +1 @@
+dst/<file>: !glob src/*

docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd

@@ -0,0 +1 @@
+uw fs copy --config-file copy.yaml --key-path config.files

docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd

@@ -0,0 +1,2 @@
+rm -rf dst/copy
+uw fs copy --report --target-dir dst/copy --config-file copy-report.yaml 2>/dev/null | jq -r .ready[]

docs/sections/user_guide/cli/tools/fs/copy-report-jq.out

@@ -0,0 +1 @@
+dst/copy/foo

docs/sections/user_guide/cli/tools/fs/copy-report.cmd

@@ -0,0 +1,2 @@
+rm -rf dst/copy-report-jq
+uw fs copy --report --target-dir dst/copy-report-jq --config-file copy-report.yaml