From ad96f15fa3ac7150ab5a4f25ce37b56de62c49ae Mon Sep 17 00:00:00 2001 From: Pedro Brochado Date: Tue, 11 Jun 2024 12:17:01 -0300 Subject: [PATCH] Make blog opt-in for faster reloads on serve --- scripts/blog-fix.py | 40 ------------------------- src/pulp_docs/cli.py | 17 +++++++++++ src/pulp_docs/data/mkdocs.yml | 2 +- src/pulp_docs/data/repolist.yml | 6 ++-- src/pulp_docs/main.py | 6 ++++ src/pulp_docs/mkdocs_macros.py | 45 ++++++++++++++++++++++++----- src/pulp_docs/navigation.py | 20 ++----------- src/pulp_docs/repository.py | 22 ++++++++------ src/pulp_docs/utils/aggregation.py | 6 ++-- staging_docs/sections/blog/index.md | 26 +---------------- staging_docs/sections/user/index.md | 7 ++--- 11 files changed, 87 insertions(+), 110 deletions(-) delete mode 100644 scripts/blog-fix.py diff --git a/scripts/blog-fix.py b/scripts/blog-fix.py deleted file mode 100644 index 0db1e4d..0000000 --- a/scripts/blog-fix.py +++ /dev/null @@ -1,40 +0,0 @@ -import re -from pathlib import Path -def read_files(): - return list(Path("staging_docs/sections/blog/posts/").glob("*.md")) - -def fix_blog_posts(): - """ - Problems: - * add heading zero to filenames missing it: yyyy-mm-d - * delete "- date:" lines from frontmatter - * add new based on filename - * quote floats that should be strings, like "- 2.0" - - """ - for file in read_files(): - this_date_split = file.name.split("-")[:3] - if len(this_date_split[-1]) == 1: - this_date_split[-1] = "0" + this_date_split[-1] - this_date = "-".join(this_date_split) - - raw = file.read_text() - # fix floats - raw = re.sub(r"\n\s+-\s*(\d+\.\d+)", r'\n - "\1"', raw, 1) - - # use aware TZ - if "- date:" not in raw: - raw = re.sub(r"---\s*\n", f"---\ndate: {this_date}T20:55:50+00:00\n", raw, 1) - - # add excerpt separator - raw = re.sub(r"\n---\s*\n", "\n---\n\n", raw, 1) - file.write_text(raw) - -def fix_floats(): - for file in read_files(): - raw = file.read_text() - raw = re.sub(r"\n -\s*(\d+\.\d+)", r'\n - "\1"', raw, 1) - file.write_text(raw) - - -fix_blog_posts() diff --git a/src/pulp_docs/cli.py b/src/pulp_docs/cli.py index ed4ba47..c8d4494 100644 --- a/src/pulp_docs/cli.py +++ b/src/pulp_docs/cli.py @@ -2,10 +2,12 @@ The main CLI module. """ +import os import typing as t from pathlib import Path import click + from pulp_docs.main import Config, PulpDocs @@ -53,6 +55,14 @@ def main(ctx: PulpDocsContext, verbose: bool): ) @click.option("--no-livereload", "livereload", flag_value=False, help=no_reload_help) @click.option("--livereload", "livereload", flag_value=True, default=True, hidden=True) +@click.option( + "-a", + "--enable-all", + "enable_all", + flag_value=True, + default=False, + help="Enable all mkdocs features (e.g, the blog rendering)", +) @pass_pulpdocs_context def serve( ctx: PulpDocsContext, @@ -60,6 +70,7 @@ def serve( verbose: bool, watch: t.List[Path], livereload: bool, + enable_all: bool, ): """Run mkdocs server.""" config = ctx.config @@ -70,6 +81,12 @@ def serve( config.watch = watch config.livereload = livereload + # by default blog should be disabled, because it affects reload time a lot. + disabled = os.environ.get("PULPDOCS_DISABLED", "blog") + if enable_all is True: + disabled = "" + config.disabled = disabled + dry_run = True if config.test_mode else False pulpdocs.serve(config, dry_run=dry_run) diff --git a/src/pulp_docs/data/mkdocs.yml b/src/pulp_docs/data/mkdocs.yml index 6b5ff0f..7a1d4c5 100644 --- a/src/pulp_docs/data/mkdocs.yml +++ b/src/pulp_docs/data/mkdocs.yml @@ -15,8 +15,8 @@ theme: - navigation.sections - navigation.path - navigation.footer - - toc.follow - navigation.top + - toc.follow - search.suggest - search.highlight - content.code.annotation diff --git a/src/pulp_docs/data/repolist.yml b/src/pulp_docs/data/repolist.yml index f6b8f69..e2df953 100644 --- a/src/pulp_docs/data/repolist.yml +++ b/src/pulp_docs/data/repolist.yml @@ -3,9 +3,9 @@ meta: rest_api_template: https://docs.pulpproject.org/{}/restapi.html repo_types: - core - - deployment - content - - interfaces + - deployment + - interaction - other repos: @@ -14,7 +14,7 @@ repos: owner: pulp title: Pulp Core branch: main - interfaces: + interaction: - name: pulp-openapi-generator owner: pulp title: OpenAPI Generator diff --git a/src/pulp_docs/main.py b/src/pulp_docs/main.py index 2970cde..8a64318 100644 --- a/src/pulp_docs/main.py +++ b/src/pulp_docs/main.py @@ -18,6 +18,10 @@ def cast_bool(value: str) -> bool: return False if value.lower() in ("f", "false") else True +def cast_list(value: str) -> t.List[str]: + return [v.strip() for v in value.split(",") if v] + + class Config: """ Configuration shared among CLI and mkdocs_macro.py hooks. @@ -38,12 +42,14 @@ def __init__(self, from_environ: bool = False): if env_mkdocs := os.environ.get("PULPDOCS_MKDOCS_FILE"): self.mkdocs_file = Path(env_mkdocs) + self.disabled = [] else: self.verbose = cast_bool(os.environ["PULPDOCS_VERBOSE"]) self.workdir = Path(os.environ["PULPDOCS_WORKDIR"]) self.mkdocs_file = Path(os.environ["PULPDOCS_MKDOCS_FILE"]) self.repolist = Path(os.environ["PULPDOCS_REPOLIST"]) self.clear_cache = cast_bool(os.environ["PULPDOCS_CLEAR_CACHE"]) + self.disabled = cast_list(os.environ.get("PULPDOCS_DISABLED", "")) self.watch: list[Path] = [] self.livereload = True self.test_mode = cast_bool(os.environ.get("PULPDOCS_TEST_MODE", "f")) diff --git a/src/pulp_docs/mkdocs_macros.py b/src/pulp_docs/mkdocs_macros.py index aab1df5..71eef60 100644 --- a/src/pulp_docs/mkdocs_macros.py +++ b/src/pulp_docs/mkdocs_macros.py @@ -21,14 +21,15 @@ import tempfile import time from pathlib import Path +from textwrap import dedent import httpx import rich from pulp_docs.cli import Config -from pulp_docs.constants import RESTAPI_URL_TEMPLATE from pulp_docs.navigation import get_navigation from pulp_docs.repository import Repo, Repos, SubPackage +from pulp_docs.utils.general import get_git_ignored_files # the name of the docs in the source repositories SRC_DOCS_DIRNAME = "staging_docs" @@ -103,7 +104,11 @@ def prepare_repositories(TMPDIR: Path, repos: Repos, config: Config): this_docs_dir = repo_docs / repo_or_pkg.name if not isinstance(repo_or_pkg, SubPackage): this_src_dir = repo_sources / repo_or_pkg.name - repo_or_pkg.download(dest_dir=this_src_dir, clear_cache=config.clear_cache) + repo_or_pkg.download( + dest_dir=this_src_dir, + clear_cache=config.clear_cache, + disabled=config.disabled, + ) else: this_src_dir = repo_sources / repo_or_pkg.subpackage_of / repo_or_pkg.name @@ -175,7 +180,10 @@ def _place_doc_files(src_dir: Path, docs_dir: Path, repo: Repo, api_src_dir: Pat log.info(f"Moving doc files:\nfrom '{src_dir}'\nto '{docs_dir}'") try: - shutil.copytree(src_dir / SRC_DOCS_DIRNAME, docs_dir / "docs") + shutil.copytree( + src_dir / SRC_DOCS_DIRNAME, + docs_dir / "docs", + ) except FileNotFoundError: Path(docs_dir / "docs").mkdir(parents=True) repo.status.has_staging_docs = False @@ -337,17 +345,30 @@ def define_env(env): exist_ok=True ) - env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"][ - "paths" - ] = code_sources + try: + env.conf["plugins"]["mkdocstrings"].config["handlers"]["python"][ + "paths" + ] = code_sources + except KeyError: + ... # Configure navigation log.info("[pulp-docs] Configuring navigation") env.conf["docs_dir"] = docs_dir env.conf["nav"] = get_navigation(docs_dir, repos) + # Disable plugins if in edit_mode + mkdocs_features = { + # "plugins": "mkdocstrings", + # "tags": "material/tags", + "blog": "material/blog", + # "search": "material/search", + } + env.conf["mkdocs_features"] = mkdocs_features + for feature_id, plugin_name in mkdocs_features.items(): + if feature_id in config.disabled: + env.conf["plugins"][plugin_name].config["enabled"] = False - # env.conf["plugins"]["material/blog"].config["blog_dir"] = "pulp-docs/docs/sections/blog" # Try to watch CWD/staging_docs watched_workdir = Path("staging_docs") if watched_workdir.exists(): @@ -412,3 +433,13 @@ def on_post_build(env): """The mkdocs-marcros 'on_post_build' hook. Used to print summary report for end user.""" # Log relevant most useful information for end-user print_user_repo(repos=env.conf["pulp_repos"], config=env.conf["pulp_config"]) + + # Print help about disabling features + features_msg = """\ + Blog rendering is disabled for faster reloads. + To enable it, use: -a, --enable-all + """ + + disabled = env.conf["pulp_config"].disabled + log.warning(f"[pulp-docs] Disabled features: {disabled}") + print(features_msg) diff --git a/src/pulp_docs/navigation.py b/src/pulp_docs/navigation.py index eb0e7c4..0fd88f6 100644 --- a/src/pulp_docs/navigation.py +++ b/src/pulp_docs/navigation.py @@ -65,29 +65,13 @@ def grouped_by_persona(tmpdir: Path, repos: Repos): manual_nav = { "user": [ {"Overview": f"{SECTION_HOST}/docs/sections/user/index.md"}, + *f.repo_grouping(TEMPLATE_STR, personas=["user", "admin"]) ], "dev": [ {"Overview": f"{SECTION_HOST}/docs/sections/dev/index.md"}, - # *f.repo_grouping(DEV_TEMPLATE_STR, personas=["dev"]), - # { - # "Core": f.repo_grouping( - # DEV_TEMPLATE_STR, repo_types=["core"], personas=["dev"] - # ) - # }, - # { - # "Plugins": f.repo_grouping( - # DEV_TEMPLATE_STR, repo_types=["content"], personas=["dev"] - # ) - # }, - # { - # "Extras": f.repo_grouping( - # DEV_TEMPLATE_STR, repo_types=["other"], personas=["dev"] - # ) - # }, + *f.repo_grouping(TEMPLATE_STR, personas=["dev"]) ], } - manual_nav["user"].extend(f.repo_grouping(TEMPLATE_STR, personas=["user", "admin"])) - manual_nav["dev"].extend(f.repo_grouping(TEMPLATE_STR, personas=["dev"])) # Custom help section help_section = [ diff --git a/src/pulp_docs/repository.py b/src/pulp_docs/repository.py index 4d517ba..757f189 100644 --- a/src/pulp_docs/repository.py +++ b/src/pulp_docs/repository.py @@ -6,9 +6,7 @@ from __future__ import annotations -import json import logging -import os import shutil import subprocess import tarfile @@ -70,11 +68,9 @@ class Repo: def __post_init__(self): self.branch_in_use = self.branch_in_use or self.branch - @property - def rest_api_link(self): - return RESTAPI_TEMPLATE.format(self.name) - - def download(self, dest_dir: Path, clear_cache: bool = False) -> str: + def download( + self, dest_dir: Path, clear_cache: bool = False, disabled: t.Sequence[str] = [] + ) -> str: """ Download repository source from url into the {dest_dir} Path. @@ -130,6 +126,12 @@ def download(self, dest_dir: Path, clear_cache: bool = False) -> str: # ignore files lisetd in .gitignore and files that starts with "." ignore_patterns = get_git_ignored_files(Path(src_copy_path)) + [".*"] + + # skip blog for faster reloads + if self.name == "pulp-docs" and "blog" in disabled: + # limitation: https://github.com/Miserlou/Zappa/issues/692#issuecomment-283012663 + ignore_patterns.append("*posts") + shutil.copytree( src_copy_path, dest_dir, @@ -243,12 +245,14 @@ def repo_types(self): @property def all(self): """The set of repositories and subpackages""" - repos = [repo for repo_type in self.repo_by_types.values() for repo in repo_type] + repos = [ + repo for repo_type in self.repo_by_types.values() for repo in repo_type + ] subpackages = [] for repo in repos: if repo.subpackages: subpackages.extend(repo.subpackages) - return repos + subpackages + return sorted(repos + subpackages, key=lambda x: x.title) def get_repos(self, repo_types: t.Optional[t.List] = None): """Get a set of repositories and subpackages by type.""" diff --git a/src/pulp_docs/utils/aggregation.py b/src/pulp_docs/utils/aggregation.py index 265c6d4..81180d2 100644 --- a/src/pulp_docs/utils/aggregation.py +++ b/src/pulp_docs/utils/aggregation.py @@ -73,13 +73,13 @@ def repo_grouping( ] selected_repo_types = repo_types or self.repos.repo_types - selected_repos = self.repos.get_repos(repo_types=selected_repo_types) + selected_repo_types = [(name, name.title()) for name in selected_repo_types] selected_personas = personas or ("user", "admin", "dev") # Create navigation main_nav = [] - for repo_type in selected_repo_types: + for repo_type, repo_type_title in selected_repo_types: repo_type_nav = [] for repo in self.repos.get_repos([repo_type]): # filter dev_only repos @@ -140,7 +140,7 @@ def repo_grouping( if repo_nav: repo_type_nav.append({repo.title: repo_nav}) if repo_type_nav: - main_nav.append({repo_type: repo_type_nav}) + main_nav.append({repo_type_title: repo_type_nav}) return main_nav or ["#"] def changes_grouping( diff --git a/staging_docs/sections/blog/index.md b/staging_docs/sections/blog/index.md index 5ffb536..e757d9b 100644 --- a/staging_docs/sections/blog/index.md +++ b/staging_docs/sections/blog/index.md @@ -3,29 +3,5 @@ hide: - toc --- -# Blog {.hide-h1} +# Pulp's Blog -
- -## Welcome to Pulp's Blog - -Here you'll find cool post about our project. - -
- -- **Featuring** - - --- - - Complete the starter tutorial to learn in pratice the basics of managing content in Pulp. - - -- **Write a Post** - - --- - - Pulp has a rich ecosystem of plugins. - Understanding the docs can help you find information more efficiently. - -
-
diff --git a/staging_docs/sections/user/index.md b/staging_docs/sections/user/index.md index 4e2dbe9..3c85b89 100644 --- a/staging_docs/sections/user/index.md +++ b/staging_docs/sections/user/index.md @@ -8,11 +8,10 @@ hide:
-## This section is for **users** - -Common needs are to *create, sync, publish and interact with repositories*. - +## This section is for **users** and **admins** +Common **user** needs are to *create, sync, publish and interact with repositories*, +while **admin** needs are to *configure, deploy and maintain Pulp instances*.