diff --git a/docs/sections/user_guide/cli/tools/config/validate-verbose.out b/docs/sections/user_guide/cli/tools/config/validate-verbose.out index 01d841518..4f07e4e73 100644 --- a/docs/sections/user_guide/cli/tools/config/validate-verbose.out +++ b/docs/sections/user_guide/cli/tools/config/validate-verbose.out @@ -1,21 +1,21 @@ -[2025-01-15T20:22:34] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose -[2025-01-15T20:22:34] DEBUG [dereference] Dereferencing, current value: -[2025-01-15T20:22:34] DEBUG [dereference] values: -[2025-01-15T20:22:34] DEBUG [dereference] greeting: Hello -[2025-01-15T20:22:34] DEBUG [dereference] recipient: World -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: values -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: values -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: greeting -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: greeting -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: Hello -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: Hello -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: recipient -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: recipient -[2025-01-15T20:22:34] DEBUG [dereference] Rendering: World -[2025-01-15T20:22:34] DEBUG [dereference] Rendered: World -[2025-01-15T20:22:34] DEBUG [dereference] Dereferencing, final value: -[2025-01-15T20:22:34] DEBUG [dereference] values: -[2025-01-15T20:22:34] DEBUG [dereference] greeting: Hello -[2025-01-15T20:22:34] DEBUG [dereference] recipient: World -[2025-01-15T20:22:34] DEBUG Using schema file: schema.jsonschema -[2025-01-15T20:22:34] INFO 0 schema-validation errors found in config +[2025-02-15T07:02:49] DEBUG Command: uw config validate --schema-file schema.jsonschema --input-file values.yaml --verbose +[2025-02-15T07:02:49] DEBUG [dereference] Dereferencing, current value: +[2025-02-15T07:02:49] DEBUG [dereference] values: +[2025-02-15T07:02:49] DEBUG [dereference] greeting: Hello +[2025-02-15T07:02:49] DEBUG [dereference] recipient: World +[2025-02-15T07:02:49] DEBUG [dereference] Rendering: values +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: values +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: greeting +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: greeting +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: Hello +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: Hello +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: recipient +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: recipient +[2025-02-15T07:02:50] DEBUG [dereference] Rendering: World +[2025-02-15T07:02:50] DEBUG [dereference] Rendered: World +[2025-02-15T07:02:50] DEBUG [dereference] Dereferencing, final value: +[2025-02-15T07:02:50] DEBUG [dereference] values: +[2025-02-15T07:02:50] DEBUG [dereference] greeting: Hello +[2025-02-15T07:02:50] DEBUG [dereference] recipient: World +[2025-02-15T07:02:50] DEBUG Validating config against external schema file: schema.jsonschema +[2025-02-15T07:02:50] INFO 0 schema-validation errors found in config diff --git a/docs/sections/user_guide/cli/tools/fs.rst b/docs/sections/user_guide/cli/tools/fs.rst index ad4321d5c..58e9ae705 100644 --- a/docs/sections/user_guide/cli/tools/fs.rst +++ b/docs/sections/user_guide/cli/tools/fs.rst @@ -27,124 +27,91 @@ Source paths prefixed with ``http://`` or ``https://`` will be copied from their Examples ^^^^^^^^ -Given ``copy-config.yaml`` containing a mapping from local-filesystem destination paths to source paths +Given a config containing a mapping from local-filesystem destination paths to source paths -.. literalinclude:: fs/copy-config.yaml +.. literalinclude:: fs/copy.yaml :language: yaml -.. literalinclude:: fs/copy-exec.cmd +.. literalinclude:: fs/copy.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec.out +.. literalinclude:: fs/copy.out :language: text Here, ``foo`` and ``bar`` are copies of their respective local-filesystem source files, and ``gpl`` is a copy of the upstream network source. The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/copy-config-timedep.yaml +.. literalinclude:: fs/copy-timedep.yaml :language: yaml -.. literalinclude:: fs/copy-exec-timedep.cmd +.. literalinclude:: fs/copy-timedep.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-timedep.out +.. literalinclude:: fs/copy-timedep.out :language: text The ``--target-dir`` option need not be specified when all destination paths are absolute, and will never be applied to absolute destination paths. If any destination paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/copy-config.yaml +.. literalinclude:: fs/copy.yaml :language: yaml -.. literalinclude:: fs/copy-exec-no-target-dir-err.cmd +.. literalinclude:: fs/copy-no-target-dir-err.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/copy-exec-no-target-dir-err.out +.. literalinclude:: fs/copy-no-target-dir-err.out :language: text When the ``--report`` option is specified, a report of files not copied ("not-ready") and copied ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: -.. literalinclude:: fs/copy-config-report.yaml +.. literalinclude:: fs/copy-report.yaml :language: yaml -.. literalinclude:: fs/copy-exec-report.cmd +.. literalinclude:: fs/copy-report.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-report.out +.. literalinclude:: fs/copy-report.out :language: text Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: -.. literalinclude:: fs/copy-exec-report-jq.cmd +.. literalinclude:: fs/copy-report-jq.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/copy-exec-report-jq.out +.. literalinclude:: fs/copy-report-jq.out :language: text -``link`` --------- - -The ``link`` action stages files in a target directory by linking files, directories, or other symbolic links. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`file block `. - -.. literalinclude:: fs/link-help.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fs/link-help.out - :language: text - -Examples -^^^^^^^^ - -Given ``link-config.yaml`` containing +Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern: -.. literalinclude:: fs/link-config.yaml +.. literalinclude:: fs/copy-glob.yaml :language: yaml -.. literalinclude:: fs/link-exec.cmd +.. literalinclude:: fs/copy-glob.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/link-exec.out +.. literalinclude:: fs/copy-glob.out :language: text -Here, ``foo`` and ``bar`` are symbolic links. +The ``--report`` output can be especially useful in combination with wildcards to allow downstream logic to process a set of copied files whose identity is not known in advance. -The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: +Note that directories are excluded, and recursive copies are not supported. To recursively copy from a shell script, where you might otherwise use ``uw fs copy``, consider using ``cp -r``; and from Python code where you might otherwise call ``uwtools.api.fs.copy()``, consider using the standard-library :python:`shutil.copytree `. For more control, including file-grained include and exclude, consider the unrivaled `rsync `_, which can be installed from conda in case your system does not already provide it. It can be called from shell scripts, or via :python:`subprocess ` from Python. -.. literalinclude:: fs/link-config-timedep.yaml - :language: yaml -.. literalinclude:: fs/link-exec-timedep.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-timedep.out - :language: text - -The ``--target-dir`` option need not be specified when all linkname paths are absolute, and will never be applied to absolute linkname paths. If any linkname paths are relative, however, it is an error not to provide a target directory: - -.. literalinclude:: fs/link-config.yaml - :language: yaml -.. literalinclude:: fs/link-exec-no-target-dir-err.cmd - :language: text - :emphasize-lines: 1 -.. literalinclude:: fs/link-exec-no-target-dir-err.out - :language: text +See :ref:`files_yaml` for more information on the semantics of the ``!glob`` tag and wildcard copies. -When the ``--report`` option is specified, a report of files not linked ("not-ready") and linked ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: +``link`` +-------- -.. literalinclude:: fs/link-config-report.yaml - :language: yaml -.. literalinclude:: fs/link-exec-report.cmd - :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-report.out - :language: text +The ``link`` action stages items in a target directory by creating symbolic links to files, directories, or other symbolic links. It otherwise behaves similarly to ``copy`` (see above), but note the following: -Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: +* In addition to file, directories and other symbolic links can be linked. +* HTTP(S) sources are not supported. +* Support for wildcard source values is the same as for ``link``. -.. literalinclude:: fs/link-exec-report-jq.cmd +.. literalinclude:: fs/link-help.cmd :language: text - :emphasize-lines: 2 -.. literalinclude:: fs/link-exec-report-jq.out + :emphasize-lines: 1 +.. literalinclude:: fs/link-help.out :language: text ``makedirs`` ------------ -The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `. +The ``makedirs`` action creates directories. Any ``KEY`` positional arguments are used to navigate, in the order given, from the top of the config to the :ref:`makedirs block `, which must nest under a ``makedirs:`` key. .. literalinclude:: fs/makedirs-help.cmd :language: text @@ -155,50 +122,34 @@ The ``makedirs`` action creates directories. Any ``KEY`` positional arguments ar Examples ^^^^^^^^ -Given ``makedirs-config.yaml`` containing +Given a config containing -.. literalinclude:: fs/makedirs-config.yaml +.. literalinclude:: fs/makedirs.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec.cmd +.. literalinclude:: fs/makedirs.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-exec.out +.. literalinclude:: fs/makedirs.out :language: text The ``--cycle`` and ``--leadtime`` options can be used to make Python ``datetime`` and ``timedelta`` objects, respectively, available for use in Jinja2 expression in the config. For example: -.. literalinclude:: fs/makedirs-config-timedep.yaml +.. literalinclude:: fs/makedirs-timedep.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec-timedep.cmd +.. literalinclude:: fs/makedirs-timedep.cmd :language: text :emphasize-lines: 2 -.. literalinclude:: fs/makedirs-exec-timedep.out +.. literalinclude:: fs/makedirs-timedep.out :language: text The ``--target-dir`` option need not be specified when all directory paths are absolute, and will never be applied to absolute paths. If any paths are relative, however, it is an error not to provide a target directory: -.. literalinclude:: fs/makedirs-config.yaml +.. literalinclude:: fs/makedirs.yaml :language: yaml -.. literalinclude:: fs/makedirs-exec-no-target-dir-err.cmd +.. literalinclude:: fs/makedirs-no-target-dir-err.cmd :language: text :emphasize-lines: 1 -.. literalinclude:: fs/makedirs-exec-no-target-dir-err.out +.. literalinclude:: fs/makedirs-no-target-dir-err.out :language: text -When the ``--report`` option is specified, a report of directories not created ("not-ready") and created ("ready") will be printed to ``stdout`` as machine-readable JSON. For example, using a config specifying both available and unavailable source files: - -.. literalinclude:: fs/makedirs-config-report.yaml - :language: yaml -.. literalinclude:: fs/makedirs-exec-report.cmd - :language: text - :emphasize-lines: 5 -.. literalinclude:: fs/makedirs-exec-report.out - :language: text - -Since ``uwtools`` logs to ``stderr``, log and report output can be separated and the latter processed with a tool like ``jq``: - -.. literalinclude:: fs/makedirs-exec-report-jq.cmd - :language: text - :emphasize-lines: 5 -.. literalinclude:: fs/makedirs-exec-report-jq.out - :language: text +The ``--report`` option behaves the same as for ``link`` (see above). diff --git a/docs/sections/user_guide/cli/tools/fs/.gitignore b/docs/sections/user_guide/cli/tools/fs/.gitignore index e49682489..3a41be37d 100644 --- a/docs/sections/user_guide/cli/tools/fs/.gitignore +++ b/docs/sections/user_guide/cli/tools/fs/.gitignore @@ -1,7 +1 @@ -copy-dst -copy-dst-report-jq -copy-dst-timedep -link-dst -link-dst-report -link-dst-report-jq -link-dst-timedep +dst/ diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd deleted file mode 100644 index d89130592..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs copy --config-file copy-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.cmd deleted file mode 100644 index d1362690c..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf copy-dst -uw fs copy --report --target-dir copy-dst --config-file copy-config-report.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.out deleted file mode 100644 index 7082d6a3a..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-report-jq.out +++ /dev/null @@ -1 +0,0 @@ -copy-dst/foo diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-report.cmd deleted file mode 100644 index 7f1b614a5..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf copy-dst-report-jq -uw fs copy --report --target-dir copy-dst-report-jq --config-file copy-config-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-report.out deleted file mode 100644 index 6377f3b55..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-report.out +++ /dev/null @@ -1,21 +0,0 @@ -[2025-01-30T06:36:44] INFO Validating config against internal schema: files-to-stage -[2025-01-30T06:36:44] INFO 0 schema-validation errors found in fs config -[2025-01-30T06:36:48] INFO File src/foo: Ready -[2025-01-30T06:36:48] WARNING File src/qux: Not ready [external asset] -[2025-01-30T06:36:48] INFO Copy src/foo -> copy-dst-report-jq/foo: Executing -[2025-01-30T06:36:48] INFO Copy src/foo -> copy-dst-report-jq/foo: Ready -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: Not ready -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: Requires: -[2025-01-30T06:36:48] WARNING Copy src/qux -> copy-dst-report-jq/qux: ✖ File src/qux -[2025-01-30T06:36:48] WARNING File copies: Not ready -[2025-01-30T06:36:48] WARNING File copies: Requires: -[2025-01-30T06:36:48] WARNING File copies: ✔ Copy src/foo -> copy-dst-report-jq/foo -[2025-01-30T06:36:48] WARNING File copies: ✖ Copy src/qux -> copy-dst-report-jq/qux -{ - "not-ready": [ - "copy-dst-report-jq/qux" - ], - "ready": [ - "copy-dst-report-jq/foo" - ] -} diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd deleted file mode 100644 index 6c3c956af..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst-timedep -uw fs copy --target-dir copy-dst-timedep --config-file copy-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files -echo -tree copy-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out deleted file mode 100644 index c7a4e8ff6..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec-timedep.out +++ /dev/null @@ -1,11 +0,0 @@ -[2024-11-23T00:30:43] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:43] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO File src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Executing -[2024-11-23T00:30:44] INFO Copy src/20240529/12/006/baz -> copy-dst-timedep/baz-2024-05-29T18: Ready -[2024-11-23T00:30:44] INFO File copies: Ready - -copy-dst-timedep -└── baz-2024-05-29T18 - -1 directory, 1 file diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd b/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd deleted file mode 100644 index f0579f20e..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf copy-dst -uw fs copy --target-dir copy-dst --config-file copy-config.yaml --key-path config.files -echo -tree copy-dst diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec.out b/docs/sections/user_guide/cli/tools/fs/copy-exec.out deleted file mode 100644 index bd0fee1c0..000000000 --- a/docs/sections/user_guide/cli/tools/fs/copy-exec.out +++ /dev/null @@ -1,21 +0,0 @@ -[2025-01-18T07:29:30] INFO Validating config against internal schema: files-to-stage -[2025-01-18T07:29:30] INFO 0 schema-validation errors found in fs config -[2025-01-18T07:29:32] INFO File src/foo: Ready -[2025-01-18T07:29:33] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready -[2025-01-18T07:29:33] INFO File src/bar: Ready -[2025-01-18T07:29:33] INFO Copy src/foo -> copy-dst/foo: Executing -[2025-01-18T07:29:33] INFO Copy src/foo -> copy-dst/foo: Ready -[2025-01-18T07:29:33] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Executing -[2025-01-18T07:29:33] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> copy-dst/licenses/gpl: Ready -[2025-01-18T07:29:33] INFO Copy src/bar -> copy-dst/subdir/bar: Executing -[2025-01-18T07:29:33] INFO Copy src/bar -> copy-dst/subdir/bar: Ready -[2025-01-18T07:29:33] INFO File copies: Ready - -copy-dst -├── foo -├── licenses -│   └── gpl -└── subdir - └── bar - -3 directories, 3 files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.cmd b/docs/sections/user_guide/cli/tools/fs/copy-glob.cmd new file mode 100644 index 000000000..ce00e70f2 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy-glob +uw fs copy --report --target-dir dst/copy-glob --config-file copy-glob.yaml +echo +tree dst/copy-glob diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.out b/docs/sections/user_guide/cli/tools/fs/copy-glob.out new file mode 100644 index 000000000..49c3b7f45 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.out @@ -0,0 +1,24 @@ +[2025-02-15T18:52:47] INFO Validating config against internal schema: files-to-stage +[2025-02-15T18:52:47] INFO 0 schema-validation errors found in fs config +[2025-02-15T18:52:47] WARNING Ignoring directory src/20240529 +[2025-02-15T18:52:47] INFO File src/foo: Ready +[2025-02-15T18:52:47] INFO File src/bar: Ready +[2025-02-15T18:52:47] INFO Copy src/foo -> dst/copy-glob/dst/foo: Executing +[2025-02-15T18:52:47] INFO Copy src/foo -> dst/copy-glob/dst/foo: Ready +[2025-02-15T18:52:47] INFO Copy src/bar -> dst/copy-glob/dst/bar: Executing +[2025-02-15T18:52:47] INFO Copy src/bar -> dst/copy-glob/dst/bar: Ready +[2025-02-15T18:52:47] INFO File copies: Ready +{ + "not-ready": [], + "ready": [ + "dst/copy-glob/dst/foo", + "dst/copy-glob/dst/bar" + ] +} + +dst/copy-glob +└── dst + ├── bar + └── foo + +2 directories, 2 files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-glob.yaml b/docs/sections/user_guide/cli/tools/fs/copy-glob.yaml new file mode 100644 index 000000000..c3f256dd0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-glob.yaml @@ -0,0 +1 @@ +dst/: !glob src/* diff --git a/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd new file mode 100644 index 000000000..90e82985d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs copy --config-file copy.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-exec-no-target-dir-err.out rename to docs/sections/user_guide/cli/tools/fs/copy-no-target-dir-err.out diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd new file mode 100644 index 000000000..d1d1f51a5 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.cmd @@ -0,0 +1,2 @@ +rm -rf dst/copy +uw fs copy --report --target-dir dst/copy --config-file copy-report.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report-jq.out b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.out new file mode 100644 index 000000000..04ec58c5d --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-report-jq.out @@ -0,0 +1 @@ +dst/copy/foo diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report.cmd b/docs/sections/user_guide/cli/tools/fs/copy-report.cmd new file mode 100644 index 000000000..018eab7c9 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-report.cmd @@ -0,0 +1,2 @@ +rm -rf dst/copy-report-jq +uw fs copy --report --target-dir dst/copy-report-jq --config-file copy-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-report.out b/docs/sections/user_guide/cli/tools/fs/copy-report.out new file mode 100644 index 000000000..0367d954a --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-report.out @@ -0,0 +1,21 @@ +[2025-02-15T17:50:50] INFO Validating config against internal schema: files-to-stage +[2025-02-15T17:50:51] INFO 0 schema-validation errors found in fs config +[2025-02-15T17:50:53] INFO File src/foo: Ready +[2025-02-15T17:50:53] WARNING File src/qux: Not ready [external asset] +[2025-02-15T17:50:53] INFO Copy src/foo -> dst/copy-report-jq/foo: Executing +[2025-02-15T17:50:53] INFO Copy src/foo -> dst/copy-report-jq/foo: Ready +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: Not ready +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: Requires: +[2025-02-15T17:50:53] WARNING Copy src/qux -> dst/copy-report-jq/qux: ✖ File src/qux +[2025-02-15T17:50:53] WARNING File copies: Not ready +[2025-02-15T17:50:53] WARNING File copies: Requires: +[2025-02-15T17:50:53] WARNING File copies: ✔ Copy src/foo -> dst/copy-report-jq/foo +[2025-02-15T17:50:53] WARNING File copies: ✖ Copy src/qux -> dst/copy-report-jq/qux +{ + "not-ready": [ + "dst/copy-report-jq/qux" + ], + "ready": [ + "dst/copy-report-jq/foo" + ] +} diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/copy-report.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config-report.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd new file mode 100644 index 000000000..f1e0ea729 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-timedep.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy-timedep +uw fs copy --target-dir dst/copy-timedep --config-file copy-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files +echo +tree dst/copy-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/copy-timedep.out b/docs/sections/user_guide/cli/tools/fs/copy-timedep.out new file mode 100644 index 000000000..8be3e9ead --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy-timedep.out @@ -0,0 +1,11 @@ +[2025-02-15T17:50:54] INFO Validating config against internal schema: files-to-stage +[2025-02-15T17:50:54] INFO 0 schema-validation errors found in fs config +[2025-02-15T17:50:56] INFO File src/20240529/12/006/baz: Ready +[2025-02-15T17:50:56] INFO Copy src/20240529/12/006/baz -> dst/copy-timedep/baz-2024-05-29T18: Executing +[2025-02-15T17:50:56] INFO Copy src/20240529/12/006/baz -> dst/copy-timedep/baz-2024-05-29T18: Ready +[2025-02-15T17:50:56] INFO File copies: Ready + +dst/copy-timedep +└── baz-2024-05-29T18 + +1 directory, 1 file diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/copy-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/copy-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/copy.cmd b/docs/sections/user_guide/cli/tools/fs/copy.cmd new file mode 100644 index 000000000..02dacbcc0 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy.cmd @@ -0,0 +1,4 @@ +rm -rf dst/copy +uw fs copy --target-dir dst/copy --config-file copy.yaml --key-path config.files +echo +tree dst/copy diff --git a/docs/sections/user_guide/cli/tools/fs/copy.out b/docs/sections/user_guide/cli/tools/fs/copy.out new file mode 100644 index 000000000..165d61afe --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/copy.out @@ -0,0 +1,19 @@ +[2025-02-15T19:08:19] INFO Validating config against internal schema: files-to-stage +[2025-02-15T19:08:19] INFO 0 schema-validation errors found in fs config +[2025-02-15T19:08:21] INFO Copy src/foo -> dst/copy/foo: Ready +[2025-02-15T19:08:21] INFO Remote object https://www.gnu.org/licenses/gpl-3.0.txt: Ready +[2025-02-15T19:08:21] INFO File src/bar: Ready +[2025-02-15T19:08:21] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Executing +[2025-02-15T19:08:21] INFO Copy https://www.gnu.org/licenses/gpl-3.0.txt -> dst/copy/licenses/gpl: Ready +[2025-02-15T19:08:21] INFO Copy src/bar -> dst/copy/subdir/bar: Executing +[2025-02-15T19:08:21] INFO Copy src/bar -> dst/copy/subdir/bar: Ready +[2025-02-15T19:08:21] INFO File copies: Ready + +dst/copy +├── foo +├── licenses +│   └── gpl +└── subdir + └── bar + +3 directories, 3 files diff --git a/docs/sections/user_guide/cli/tools/fs/copy-config.yaml b/docs/sections/user_guide/cli/tools/fs/copy.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/copy-config.yaml rename to docs/sections/user_guide/cli/tools/fs/copy.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml deleted file mode 100644 index 30679bb0a..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config-report.yaml +++ /dev/null @@ -1,2 +0,0 @@ -foo: src/foo -qux: src/qux diff --git a/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml deleted file mode 100644 index 2a16449d7..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config-timedep.yaml +++ /dev/null @@ -1,7 +0,0 @@ -config: - files: - baz-{{ validtime }}: src/{{ yyyymmdd }}/{{ hh }}/{{ nnn }}/baz -yyyymmdd: "{{ cycle.strftime('%Y%m%d') }}" -hh: "{{ cycle.strftime('%H') }}" -nnn: "{{ '%03d' % (leadtime.total_seconds() // 3600) }}" -validtime: "{{ (cycle + leadtime).strftime('%Y-%m-%dT%H') }}" diff --git a/docs/sections/user_guide/cli/tools/fs/link-config.yaml b/docs/sections/user_guide/cli/tools/fs/link-config.yaml deleted file mode 100644 index 17c45a3e6..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-config.yaml +++ /dev/null @@ -1,4 +0,0 @@ -config: - files: - foo: src/foo - subdir/bar: src/bar diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd deleted file mode 100644 index d1d03c79b..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs link --config-file link-config.yaml --key-path config.files diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out deleted file mode 100644 index 13a0a6e63..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-no-target-dir-err.out +++ /dev/null @@ -1,3 +0,0 @@ -[2024-12-07T01:01:55] INFO Validating config against internal schema: files-to-stage -[2024-12-07T01:01:55] INFO 0 schema-validation errors found in fs config -[2024-12-07T01:01:55] ERROR Relative path 'foo' requires target directory to be specified diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd deleted file mode 100644 index 1b2f07d3f..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf link-dst-report-jq -uw fs link --report --target-dir link-dst-report-jq --config-file link-config-report.yaml 2>/dev/null | jq -r .ready[] diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out deleted file mode 100644 index de75e07d1..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report-jq.out +++ /dev/null @@ -1 +0,0 @@ -link-dst-report-jq/foo diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd deleted file mode 100644 index bafa22fe8..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report.cmd +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf link-dst-report -uw fs link --report --target-dir link-dst-report --config-file link-config-report.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-report.out b/docs/sections/user_guide/cli/tools/fs/link-exec-report.out deleted file mode 100644 index e15034975..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-report.out +++ /dev/null @@ -1,21 +0,0 @@ -[2025-01-30T06:36:37] INFO Validating config against internal schema: files-to-stage -[2025-01-30T06:36:38] INFO 0 schema-validation errors found in fs config -[2025-01-30T06:36:40] INFO Filesystem item src/foo: Ready -[2025-01-30T06:36:40] WARNING Filesystem item src/qux: Not ready [external asset] -[2025-01-30T06:36:40] INFO Link link-dst-report/foo -> src/foo: Executing -[2025-01-30T06:36:40] INFO Link link-dst-report/foo -> src/foo: Ready -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: Not ready -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: Requires: -[2025-01-30T06:36:41] WARNING Link link-dst-report/qux -> src/qux: ✖ Filesystem item src/qux -[2025-01-30T06:36:41] WARNING File links: Not ready -[2025-01-30T06:36:41] WARNING File links: Requires: -[2025-01-30T06:36:41] WARNING File links: ✔ Link link-dst-report/foo -> src/foo -[2025-01-30T06:36:41] WARNING File links: ✖ Link link-dst-report/qux -> src/qux -{ - "not-ready": [ - "link-dst-report/qux" - ], - "ready": [ - "link-dst-report/foo" - ] -} diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd deleted file mode 100644 index d815b7d38..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst-timedep -uw fs link --target-dir link-dst-timedep --config-file link-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config.files -echo -tree link-dst-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out deleted file mode 100644 index 6a6649751..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec-timedep.out +++ /dev/null @@ -1,11 +0,0 @@ -[2024-11-23T00:30:44] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:44] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO Filesystem item src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Executing -[2024-11-23T00:30:44] INFO Link link-dst-timedep/baz-2024-05-29T18 -> src/20240529/12/006/baz: Ready -[2024-11-23T00:30:44] INFO File links: Ready - -link-dst-timedep -└── baz-2024-05-29T18 -> ../src/20240529/12/006/baz - -1 directory, 1 file diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd b/docs/sections/user_guide/cli/tools/fs/link-exec.cmd deleted file mode 100644 index e1ca6bcda..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.cmd +++ /dev/null @@ -1,4 +0,0 @@ -rm -rf link-dst -uw fs link --target-dir link-dst --config-file link-config.yaml --key-path config.files -echo -tree link-dst diff --git a/docs/sections/user_guide/cli/tools/fs/link-exec.out b/docs/sections/user_guide/cli/tools/fs/link-exec.out deleted file mode 100644 index 18738bc39..000000000 --- a/docs/sections/user_guide/cli/tools/fs/link-exec.out +++ /dev/null @@ -1,16 +0,0 @@ -[2024-11-23T00:30:43] INFO Validating config against internal schema: files-to-stage -[2024-11-23T00:30:43] INFO 0 schema-validation errors found in fs config -[2024-11-23T00:30:44] INFO Filesystem item src/foo: Ready -[2024-11-23T00:30:44] INFO Filesystem item src/bar: Ready -[2024-11-23T00:30:44] INFO Link link-dst/foo -> src/foo: Executing -[2024-11-23T00:30:44] INFO Link link-dst/foo -> src/foo: Ready -[2024-11-23T00:30:44] INFO Link link-dst/subdir/bar -> src/bar: Executing -[2024-11-23T00:30:44] INFO Link link-dst/subdir/bar -> src/bar: Ready -[2024-11-23T00:30:44] INFO File links: Ready - -link-dst -├── foo -> ../src/foo -└── subdir - └── bar -> ../../src/bar - -2 directories, 2 files diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml deleted file mode 100644 index 0fb268290..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-config-report.yaml +++ /dev/null @@ -1,3 +0,0 @@ -makedirs: - - foo - - subdir/bar diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd deleted file mode 100644 index 8c2d83cc9..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.cmd +++ /dev/null @@ -1 +0,0 @@ -uw fs makedirs --config-file makedirs-config.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd deleted file mode 100644 index 99a4b194b..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.cmd +++ /dev/null @@ -1,6 +0,0 @@ -d=makedirs-parent-report-jq -rm -rf $d -mkdir -p $d/subdir -chmod 550 $d/subdir # read-only -uw fs makedirs --report --target-dir $d --config-file makedirs-config-report.yaml 2>/dev/null | jq -r .ready[] -chmod 750 $d/subdir # read-write diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out deleted file mode 100644 index 596ea9e41..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report-jq.out +++ /dev/null @@ -1 +0,0 @@ -makedirs-parent-report-jq/foo diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd deleted file mode 100644 index 6842f2662..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.cmd +++ /dev/null @@ -1,6 +0,0 @@ -d=makedirs-parent-report -rm -rf $d -mkdir -p $d/subdir -chmod 550 $d/subdir # read-only -uw fs makedirs --report --target-dir $d --config-file makedirs-config-report.yaml -chmod 750 $d/subdir # read-write diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out b/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out deleted file mode 100644 index f7022e576..000000000 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-report.out +++ /dev/null @@ -1,19 +0,0 @@ -[2025-02-13T17:19:49] INFO Validating config against internal schema: makedirs -[2025-02-13T17:19:49] INFO 0 schema-validation errors found in fs config -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/foo: Executing -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/foo: Ready -[2025-02-13T17:19:50] INFO Directory makedirs-parent-report/subdir/bar: Executing -[2025-02-13T17:19:50] ERROR [Errno 13] Permission denied: 'makedirs-parent-report/subdir/bar' -[2025-02-13T17:19:50] WARNING Directory makedirs-parent-report/subdir/bar: Not ready -[2025-02-13T17:19:50] WARNING Directories: Not ready -[2025-02-13T17:19:50] WARNING Directories: Requires: -[2025-02-13T17:19:50] WARNING Directories: ✔ Directory makedirs-parent-report/foo -[2025-02-13T17:19:50] WARNING Directories: ✖ Directory makedirs-parent-report/subdir/bar -{ - "not-ready": [ - "makedirs-parent-report/subdir/bar" - ], - "ready": [ - "makedirs-parent-report/foo" - ] -} diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd new file mode 100644 index 000000000..28fc0a702 --- /dev/null +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.cmd @@ -0,0 +1 @@ +uw fs makedirs --config-file makedirs.yaml --key-path config diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out b/docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-no-target-dir-err.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-no-target-dir-err.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd similarity index 61% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd index 221bac642..1ebaaff7c 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent-timedep -uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-config-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config +uw fs makedirs --target-dir makedirs-parent-timedep --config-file makedirs-timedep.yaml --cycle 2024-05-29T12 --leadtime 6 --key-path config echo tree -F makedirs-parent-timedep diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec-timedep.out rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs-timedep.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-config-timedep.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs-timedep.yaml diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd b/docs/sections/user_guide/cli/tools/fs/makedirs.cmd similarity index 77% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd rename to docs/sections/user_guide/cli/tools/fs/makedirs.cmd index 4ca49133b..1afd2c1a4 100644 --- a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.cmd +++ b/docs/sections/user_guide/cli/tools/fs/makedirs.cmd @@ -1,4 +1,4 @@ rm -rf makedirs-parent -uw fs makedirs --target-dir makedirs-parent --config-file makedirs-config.yaml --key-path config +uw fs makedirs --target-dir makedirs-parent --config-file makedirs.yaml --key-path config echo tree -F makedirs-parent diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-exec.out b/docs/sections/user_guide/cli/tools/fs/makedirs.out similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-exec.out rename to docs/sections/user_guide/cli/tools/fs/makedirs.out diff --git a/docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml b/docs/sections/user_guide/cli/tools/fs/makedirs.yaml similarity index 100% rename from docs/sections/user_guide/cli/tools/fs/makedirs-config.yaml rename to docs/sections/user_guide/cli/tools/fs/makedirs.yaml diff --git a/docs/sections/user_guide/yaml/files.rst b/docs/sections/user_guide/yaml/files.rst index 0d34ba38d..3f98114db 100644 --- a/docs/sections/user_guide/yaml/files.rst +++ b/docs/sections/user_guide/yaml/files.rst @@ -3,6 +3,9 @@ File Blocks =========== +Basics +------ + File blocks define files to be staged in a target directory as copies or symbolic links. Keys in such blocks specify either absolute destination paths, or destination paths relative to the target directory. Values specify source paths. Example block: @@ -12,18 +15,18 @@ Example block: foo: /path/to/foo subdir/bar: /path/to/bar -* Result when copying: +* Result when copying to target directory ``target/``: .. code-block:: text - target/ + target ├── foo └── subdir └── bar where ``foo`` and ``bar`` are copies of their respective source files. -* Result when linking: +* Result when linking to target directory ``target/``: .. code-block:: text @@ -33,3 +36,72 @@ where ``foo`` and ``bar`` are copies of their respective source files. └── bar -> /path/to/bar where ``foo`` and ``bar`` are symbolic links. + +HTTP(S) Support +--------------- + +Sources values may be ``http://`` or ``https://`` URLs when copying. + +Example block: + +.. code-block:: yaml + + index: https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20241001/conus/hrrr.t01z.wrfprsf00.grib2.idx + +* Result when copying to target directory ``target/``: + +.. code-block:: text + + target + └── index + +HTTP(S) sources are not supported when linking. + +Wildcard Support +---------------- + +Use the ``!glob`` tag to specify that a source-path value should be treated as a glob pattern: + +Example block: + +.. code-block:: yaml + + a/: !glob /src/a* + b/: !glob /src/b* + +Given ``/src/`` directory + +.. code-block:: text + + src + ├── aardvark + ├── apple + ├── banana + ├── bear + ├── cheetah + └── cherry + +* Result when copying to target directory ``target/``: + +.. code-block:: text + + target/ + ├── a + │ ├── aardvark + │ └── apple + └── b + ├── banana + └── bear + +Behavior is similar when linking. + +Wildcards are not supported in combination with HTTP(S) sources. + +Note that the destination-path key is treated as a template, with the rightmost component (```` and ```` above) discarded and replaced with actual filenames. Since YAML Mapping / Python ``dict`` keys must be unique, this supports the case where the same directory is the target of multiple copies, e.g. + +.. code-block:: yaml + + /media/: !glob /some/path/*.jpg + /media/: !glob /another/path/*.mp4 + +A useful convention, adopted here, is to bracket the rightmost component between ``<`` and ``>`` characters as a visual reminder that the component is a placeholder, but this is arbitrary and the brackets have no special meaning. diff --git a/src/uwtools/config/validator.py b/src/uwtools/config/validator.py index 0e8910ded..31b378dff 100644 --- a/src/uwtools/config/validator.py +++ b/src/uwtools/config/validator.py @@ -7,12 +7,13 @@ from pathlib import Path from typing import Optional, Union -from jsonschema import Draft202012Validator +from jsonschema import Draft202012Validator, validators from jsonschema.exceptions import ValidationError from referencing import Registry, Resource from referencing.jsonschema import DRAFT202012 from uwtools.config.formats.yaml import YAMLConfig +from uwtools.config.support import UWYAMLGlob from uwtools.exceptions import UWConfigError from uwtools.logging import INDENT, log from uwtools.utils.file import resource_path @@ -150,7 +151,7 @@ def validate_external( else: config = config_data if not str(schema_file).startswith(str(resource_path())): - log.debug("Using schema file: %s", schema_file) + log.debug("Validating config against external schema file: %s", schema_file) with open(schema_file, "r", encoding="utf-8") as f: schema = json.load(f) if not validate(schema=schema, desc=desc, config=config): @@ -184,5 +185,10 @@ def _validation_errors(config: JSONValueT, schema: dict) -> list[ValidationError :param schema: JSON Schema to validate the config against. :return: Any validation errors. """ - validator = Draft202012Validator(schema, registry=_registry()) + base = Draft202012Validator + type_checker = base.TYPE_CHECKER.redefine( + "fs_src", lambda _, x: any(isinstance(x, t) for t in [str, UWYAMLGlob]) + ) + UWValidator = validators.extend(base, type_checker=type_checker) + validator = UWValidator(schema, registry=_registry()) return list(validator.iter_errors(config)) diff --git a/src/uwtools/fs.py b/src/uwtools/fs.py index 1edeeb12f..0964ac98a 100644 --- a/src/uwtools/fs.py +++ b/src/uwtools/fs.py @@ -4,6 +4,7 @@ import datetime as dt from abc import ABC, abstractmethod +from glob import glob from pathlib import Path from typing import Optional, Union from urllib.parse import urlparse @@ -11,10 +12,11 @@ from iotaa import tasks from uwtools.config.formats.yaml import YAMLConfig -from uwtools.config.support import YAMLKey +from uwtools.config.support import UWYAMLGlob, YAMLKey from uwtools.config.tools import walk_key_path from uwtools.config.validator import validate_internal from uwtools.exceptions import UWConfigError +from uwtools.logging import log from uwtools.strings import STR from uwtools.utils.api import str2path from uwtools.utils.tasks import directory, filecopy, symlink @@ -136,6 +138,27 @@ def _dst_paths(self) -> list[str]: """ return list(self._config.keys()) + def _expand_wildcards(self) -> list[tuple[str, str]]: + items = [] + for dst, src in self._config.items(): + assert isinstance(src, (str, UWYAMLGlob)) + if isinstance(src, UWYAMLGlob): + pattern = src.value + attrs = urlparse(pattern) + if attrs.scheme not in ["", "file"]: + msg = "URL scheme '%s' incompatible with tag %s in: %s" + log.error(msg, attrs.scheme, src.tag, src) + if attrs.scheme == "file": + pattern = attrs.path + for path in map(Path, glob(pattern)): + if not path.is_dir(): + items.append((str(Path(dst).parent / path.name), str(path))) + else: + log.warning("Ignoring directory %s", path) + else: + items.append((dst, src)) + return items + @property def _schema(self) -> str: """ @@ -157,7 +180,7 @@ def go(self): yield "File copies" yield [ filecopy(src=src, dst=self._simple(self._target_dir) / self._simple(dst)) - for dst, src in self._config.items() + for dst, src in self._expand_wildcards() ] @staticmethod @@ -182,7 +205,7 @@ def go(self): """ linkname = lambda k: Path(self._target_dir / k if self._target_dir else k) yield "File links" - yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._config.items()] + yield [symlink(target=Path(v), linkname=linkname(k)) for k, v in self._expand_wildcards()] class MakeDirs(Stager): diff --git a/src/uwtools/resources/jsonschema/files-to-stage.jsonschema b/src/uwtools/resources/jsonschema/files-to-stage.jsonschema index 7c83c61f3..26ca12529 100644 --- a/src/uwtools/resources/jsonschema/files-to-stage.jsonschema +++ b/src/uwtools/resources/jsonschema/files-to-stage.jsonschema @@ -1,6 +1,6 @@ { "additionalProperties": { - "type": "string" + "type": "fs_src" }, "minProperties": 1, "type": "object" diff --git a/src/uwtools/tests/test_fs.py b/src/uwtools/tests/test_fs.py index f923520bd..4d590e536 100644 --- a/src/uwtools/tests/test_fs.py +++ b/src/uwtools/tests/test_fs.py @@ -3,8 +3,10 @@ # pylint: disable=protected-access # pylint: disable=redefined-outer-name +import logging from logging import getLogger from pathlib import Path +from textwrap import dedent from unittest.mock import Mock, patch import iotaa @@ -12,7 +14,10 @@ from pytest import fixture, mark, raises from uwtools import fs +from uwtools.config.support import uw_yaml_loader from uwtools.exceptions import UWConfigError +from uwtools.logging import log +from uwtools.tests.support import logged # Fixtures @@ -53,12 +58,13 @@ def _schema(self): # Tests -@mark.parametrize("src_fn", [str, Path]) -@mark.parametrize("dst_fn", [str, Path]) -@mark.parametrize("td_fn", [str, Path]) -def test_fs_Copier_go(src_fn, dst_fn, td_fn): - src, td, dst = src_fn("/src/file"), td_fn("/dst"), dst_fn("file") - obj = Mock(_config={dst: src}, _simple=fs.Copier._simple, _target_dir=td) +@mark.parametrize("src_func", [str, Path]) +@mark.parametrize("dst_func", [str, Path]) +@mark.parametrize("tgt_func", [str, Path]) +def test_fs_Copier_go(src_func, dst_func, tgt_func): + src, dst, tgt = src_func("/src/file"), dst_func("file"), tgt_func("/dst") + obj = Mock(_simple=fs.Copier._simple, _target_dir=tgt) + obj._expand_wildcards.return_value = [(dst, src)] with patch.object(fs, "filecopy") as filecopy: filecopy.return_value = iotaa.NodeExternal( taskname="test", threads=0, logger=getLogger(), assets_=None @@ -96,7 +102,7 @@ def test_fs_Copier_go_live_no_targetdir_abspath_pass(assets): assert all(path.is_file() for path in [dstdir / "foo", dstdir / "bar"]) -def test_Copier_no_targetdir_relpath_fail(assets): +def test_Copier_go_no_targetdir_relpath_fail(assets): _, cfgdict, _ = assets with raises(UWConfigError) as e: fs.Copier(config=cfgdict, key_path=["a", "b"]).go() @@ -118,6 +124,48 @@ def test_fs_FilerStager(assets, source): assert fs.FileStager(target_dir=dstdir, config=config, key_path=["a", "b"]) +def test_fs_FileStager__expand_wildcards(caplog, tmp_path): + log.setLevel(logging.WARNING) + d = tmp_path + for fn in ["a1", "a2", "b1"]: + (d / fn).touch() + (d / "a3").mkdir() + config = f""" + /dst/: !glob {d}/a* + /dst/b1: {d}/b1 + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + assert sorted(fs.FileStager._expand_wildcards(obj)) == [ + ("/dst/a1", str(d / "a1")), + ("/dst/a2", str(d / "a2")), + ("/dst/b1", str(d / "b1")), + ] + assert logged(caplog, f"Ignoring directory {d}/a3") + + +def test_fs_FileStager__expand_wildcards_bad_scheme(caplog): + config = """ + /dst/: !glob https://foo.com/obj/* + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + assert not fs.FileStager._expand_wildcards(obj) + msg = "URL scheme 'https' incompatible with tag !glob in: !glob https://foo.com/obj/*" + assert logged(caplog, msg) + + +def test_fs_FileStager__expand_wildcards_file_scheme(): + config = """ + /dst/: !glob file:///src/a* + """ + obj = Mock(_config=yaml.load(dedent(config), Loader=uw_yaml_loader())) + with patch.object(fs, "glob", return_value=["/src/a1", "/src/a2"]) as glob: + assert fs.FileStager._expand_wildcards(obj) == [ + ("/dst/a1", "/src/a1"), + ("/dst/a2", "/src/a2"), + ] + glob.assert_called_once_with("/src/a*") + + @mark.parametrize("source", ("dict", "file")) def test_fs_Linker(assets, source): dstdir, cfgdict, cfgfile = assets diff --git a/src/uwtools/tests/test_schemas.py b/src/uwtools/tests/test_schemas.py index f49a467e9..463aa3c06 100644 --- a/src/uwtools/tests/test_schemas.py +++ b/src/uwtools/tests/test_schemas.py @@ -768,7 +768,7 @@ def test_schema_stage_files(): # An empty dict is not allowed: assert non_empty_dict(errors({})) # Non-string values are not allowed: - assert "True is not of type 'string'\n" in errors({"file1": True}) + assert "True is not of type 'fs_src'\n" in errors({"file1": True}) # filter-topo