diff --git a/docs/source/conf.py b/docs/source/conf.py index 86bb6371e2..15bc1e2873 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -19,6 +19,7 @@ import sys from pathlib import Path +sys.path.append(str(Path("./ext/illiterate_sphinx").resolve())) sys.path.append(str(Path("./ext").resolve())) # -- Project information ----------------------------------------------------- @@ -40,7 +41,7 @@ exclude_patterns = [] extensions = [ # recipe directive - "adbc_cookbook", + "illiterate_sphinx", # generic directives to enable intersphinx for java "adbc_java_domain", "numpydoc", @@ -114,6 +115,12 @@ "source_directory": "docs/source/", } +# -- Options for Illiterate Sphinx ------------------------------------------- + +illiterate_repo_url_template = ( + "https://github.com/apache/arrow-adbc/blob/main/docs/source/{rel_filename}" +) + # -- Options for Intersphinx ------------------------------------------------- intersphinx_mapping = { diff --git a/docs/source/ext/illiterate_sphinx/README.markdown b/docs/source/ext/illiterate_sphinx/README.markdown new file mode 100644 index 0000000000..95aa216433 --- /dev/null +++ b/docs/source/ext/illiterate_sphinx/README.markdown @@ -0,0 +1,73 @@ + + +# Illiterate Sphinx: Not Literate Programming, For Sphinx + +[Literate Programming][literate] is a style of programming introduced by the +venerable Donald Knuth where a program is developed as a prose document +interspersed with code. The code is then extracted from the document for +actual use. + +This isn't that. + +Illiterate Sphinx is an extension for the [Sphinx][sphinx] documentation +generator that lets you write code interspersed with special comments. The +extension then separates the code and comments to generate a prose document +interspersed with code. Meanwhile, since the source document is still just +code, it can be run and/or tested without special documentation-specific +tooling. And Sphinx doesn't need to understand anything about the source +language other than what comment delimiter it should look for. + +This extension makes it easy to write self-contained "cookbook" documentation +pages that can be run and tested as real code while still being nicely +formatted in documentation. + +## Usage + +#. Install this extension. +#. Add `illiterate_sphinx` to `extensions` in your `conf.py`. +#. Optionally, set the config value `illiterate_repo_url_template` to a URL + with a `{rel_filename}` placeholder. This will be used to link to the full + recipe source on GitHub or your favorite code hosting platform. +#. Write your cookbook recipe. For example: + + ```python + + # This is my license header. It's boring, so I don't want it to show up + # in the documentation. + + # RECIPE STARTS HERE + #: Some prose describing what I'm about to do. + #: **reStructuredText syntax works here.** + print("Hello, world!") + + #: I can have more prose now. + #: + #: - I can even write lists. + print("Goodbye, world!") + ``` + +#. Include the recipe in your documentation via the `recipe` directive: + + ```rst + .. recipe:: helloworld.py + ``` + +[literate]: https://en.wikipedia.org/wiki/Literate_programming +[sphinx]: https://www.sphinx-doc.org/en/master/ diff --git a/docs/source/ext/illiterate_sphinx/illiterate_sphinx/__init__.py b/docs/source/ext/illiterate_sphinx/illiterate_sphinx/__init__.py new file mode 100644 index 0000000000..b2b4501ec6 --- /dev/null +++ b/docs/source/ext/illiterate_sphinx/illiterate_sphinx/__init__.py @@ -0,0 +1,169 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A directive for code recipes with a literate programming style.""" + +import typing +from pathlib import Path + +import docutils +from docutils.parsers.rst import directives +from docutils.statemachine import StringList +from sphinx.util.docutils import SphinxDirective +from sphinx.util.nodes import nested_parse_with_titles +from sphinx.util.typing import OptionSpec + +__all__ = ["setup"] + + +class SourceLine(typing.NamedTuple): + content: str + lineno: int + + +class SourceFragment(typing.NamedTuple): + kind: str + lines: list[SourceLine] + + +PREAMBLE = "Recipe source: `{name} <{url}>`_" + + +class RecipeDirective(SphinxDirective): + has_content = False + required_arguments = 1 + optional_arguments = 0 + option_spec: OptionSpec = { + "language": directives.unchanged_required, + "prose-prefix": directives.unchanged_required, + } + + @staticmethod + def default_prose_prefix(language: str) -> str: + return { + "cpp": "///", + "go": "///", + "python": "#:", + }.get(language, "#:") + + def run(self): + rel_filename, filename = self.env.relfn2path(self.arguments[0]) + self.env.note_dependency(rel_filename) + self.env.note_dependency(__file__) + + language = self.options.get("language", "python") + prefix = self.options.get("prose-prefix", self.default_prose_prefix(language)) + + # --- Split the source into runs of prose or code + + fragments = [] + + fragment = [] + fragment_type = None + state = "before" + lineno = 1 + for line in open(filename): + if state == "before": + if "RECIPE STARTS HERE" in line: + state = "reading" + elif state == "reading": + if line.strip().startswith(prefix): + line_type = "prose" + # Remove prefix and next whitespace + line = line.lstrip()[len(prefix) + 1 :] + else: + line_type = "code" + + if line_type != fragment_type: + if fragment: + fragments.append( + SourceFragment(kind=fragment_type, lines=fragment) + ) + fragment = [] + fragment_type = line_type + + # Skip blank code lines + if line_type != "code" or line.strip(): + # Remove trailing newline + fragment.append(SourceLine(content=line[:-1], lineno=lineno)) + + lineno += 1 + + if fragment: + fragments.append(SourceFragment(kind=fragment_type, lines=fragment)) + + # --- Generate the final reST as a whole and parse it + # That way, section hierarchy works properly + + generated_lines = [] + + # Link to the source on GitHub + repo_url_template = self.env.config.illiterate_repo_url_template + if repo_url_template is not None: + repo_url = repo_url_template.format(rel_filename=rel_filename) + generated_lines.append( + PREAMBLE.format( + name=Path(rel_filename).name, + url=repo_url, + ) + ) + + # Paragraph break + generated_lines.append("") + + for fragment in fragments: + if fragment.kind == "prose": + generated_lines.extend([line.content for line in fragment.lines]) + generated_lines.append("") + elif fragment.kind == "code": + line_min = fragment.lines[0].lineno + line_max = fragment.lines[-1].lineno + lines = [ + f".. literalinclude:: {self.arguments[0]}", + f" :language: {language}", + " :linenos:", + " :lineno-match:", + f" :lines: {line_min}-{line_max}", + "", + ] + generated_lines.extend(lines) + else: + raise RuntimeError("Unknown fragment kind") + + parsed = docutils.nodes.Element() + nested_parse_with_titles( + self.state, + StringList(generated_lines, source=""), + parsed, + ) + return parsed.children + + +def setup(app) -> None: + app.add_config_value( + "illiterate_repo_url_template", + default=None, + rebuild="html", + types=str, + ) + app.add_directive("recipe", RecipeDirective) + + return { + "version": "0.1", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/docs/source/ext/illiterate_sphinx/pyproject.toml b/docs/source/ext/illiterate_sphinx/pyproject.toml new file mode 100644 index 0000000000..2e39df2ec6 --- /dev/null +++ b/docs/source/ext/illiterate_sphinx/pyproject.toml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[project] +name = "illiterate-sphinx" +description = "A Sphinx directive for not-quite-literate programming." +authors = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}] +license = {text = "Apache-2.0"} +readme = "README.md" +requires-python = ">=3.9" +dependencies = ["sphinx"] +version = "0.1.0" + +[project.urls] +homepage = "https://arrow.apache.org/adbc/" +repository = "https://github.com/apache/arrow-adbc" + +[build-system] +requires = ["setuptools >= 61.0.0"] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +xfail_strict = true + +[tool.setuptools] +packages = ["illiterate_sphinx"] +py-modules = ["illiterate_sphinx"]