chore: make Sphinx cookbook extension pip-installable

Fixes apache#2432.

docs/source/conf.py

@@ -19,6 +19,7 @@
 import sys
 from pathlib import Path
 
+sys.path.append(str(Path("./ext/illiterate_sphinx").resolve()))
 sys.path.append(str(Path("./ext").resolve()))
 
 # -- Project information -----------------------------------------------------
@@ -40,7 +41,7 @@
 exclude_patterns = []
 extensions = [
     # recipe directive
-    "adbc_cookbook",
+    "illiterate_sphinx",
     # generic directives to enable intersphinx for java
     "adbc_java_domain",
     "numpydoc",
@@ -114,6 +115,12 @@
     "source_directory": "docs/source/",
 }
 
+# -- Options for Illiterate Sphinx -------------------------------------------
+
+illiterate_repo_url_template = (
+    "https://github.com/apache/arrow-adbc/blob/main/docs/source/{rel_filename}"
+)
+
 # -- Options for Intersphinx -------------------------------------------------
 
 intersphinx_mapping = {

docs/source/ext/illiterate_sphinx/README.markdown

@@ -0,0 +1,73 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Illiterate Sphinx: Not Literate Programming, For Sphinx
+
+[Literate Programming][literate] is a style of programming introduced by the
+venerable Donald Knuth where a program is developed as a prose document
+interspersed with code.  The code is then extracted from the document for
+actual use.
+
+This isn't that.
+
+Illiterate Sphinx is an extension for the [Sphinx][sphinx] documentation
+generator that lets you write code interspersed with special comments.  The
+extension then separates the code and comments to generate a prose document
+interspersed with code.  Meanwhile, since the source document is still just
+code, it can be run and/or tested without special documentation-specific
+tooling.  And Sphinx doesn't need to understand anything about the source
+language other than what comment delimiter it should look for.
+
+This extension makes it easy to write self-contained "cookbook" documentation
+pages that can be run and tested as real code while still being nicely
+formatted in documentation.
+
+## Usage
+
+#. Install this extension.
+#. Add `illiterate_sphinx` to `extensions` in your `conf.py`.
+#. Optionally, set the config value `illiterate_repo_url_template` to a URL
+   with a `{rel_filename}` placeholder.  This will be used to link to the full
+   recipe source on GitHub or your favorite code hosting platform.
+#. Write your cookbook recipe.  For example:
+
+    ```python
+
+    # This is my license header. It's boring, so I don't want it to show up
+    # in the documentation.
+
+    # RECIPE STARTS HERE
+    #: Some prose describing what I'm about to do.
+    #: **reStructuredText syntax works here.**
+    print("Hello, world!")
+
+    #: I can have more prose now.
+    #:
+    #: - I can even write lists.
+    print("Goodbye, world!")
+    ```
+
+#. Include the recipe in your documentation via the `recipe` directive:
+
+    ```rst
+    .. recipe:: helloworld.py
+    ```
+
+[literate]: https://en.wikipedia.org/wiki/Literate_programming
+[sphinx]: https://www.sphinx-doc.org/en/master/

docs/source/ext/illiterate_sphinx/illiterate_sphinx/__init__.py

@@ -0,0 +1,169 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""A directive for code recipes with a literate programming style."""
+
+import typing
+from pathlib import Path
+
+import docutils
+from docutils.parsers.rst import directives
+from docutils.statemachine import StringList
+from sphinx.util.docutils import SphinxDirective
+from sphinx.util.nodes import nested_parse_with_titles
+from sphinx.util.typing import OptionSpec
+
+__all__ = ["setup"]
+
+
+class SourceLine(typing.NamedTuple):
+    content: str
+    lineno: int
+
+
+class SourceFragment(typing.NamedTuple):
+    kind: str
+    lines: list[SourceLine]
+
+
+PREAMBLE = "Recipe source: `{name} <{url}>`_"
+
+
+class RecipeDirective(SphinxDirective):
+    has_content = False
+    required_arguments = 1
+    optional_arguments = 0
+    option_spec: OptionSpec = {
+        "language": directives.unchanged_required,
+        "prose-prefix": directives.unchanged_required,
+    }
+
+    @staticmethod
+    def default_prose_prefix(language: str) -> str:
+        return {
+            "cpp": "///",
+            "go": "///",
+            "python": "#:",
+        }.get(language, "#:")
+
+    def run(self):
+        rel_filename, filename = self.env.relfn2path(self.arguments[0])
+        self.env.note_dependency(rel_filename)
+        self.env.note_dependency(__file__)
+
+        language = self.options.get("language", "python")
+        prefix = self.options.get("prose-prefix", self.default_prose_prefix(language))
+
+        # --- Split the source into runs of prose or code
+
+        fragments = []
+
+        fragment = []
+        fragment_type = None
+        state = "before"
+        lineno = 1
+        for line in open(filename):
+            if state == "before":
+                if "RECIPE STARTS HERE" in line:
+                    state = "reading"
+            elif state == "reading":
+                if line.strip().startswith(prefix):
+                    line_type = "prose"
+                    # Remove prefix and next whitespace
+                    line = line.lstrip()[len(prefix) + 1 :]
+                else:
+                    line_type = "code"
+
+                if line_type != fragment_type:
+                    if fragment:
+                        fragments.append(
+                            SourceFragment(kind=fragment_type, lines=fragment)
+                        )
+                        fragment = []
+                    fragment_type = line_type
+
+                # Skip blank code lines
+                if line_type != "code" or line.strip():
+                    # Remove trailing newline
+                    fragment.append(SourceLine(content=line[:-1], lineno=lineno))
+
+            lineno += 1
+
+        if fragment:
+            fragments.append(SourceFragment(kind=fragment_type, lines=fragment))
+
+        # --- Generate the final reST as a whole and parse it
+        # That way, section hierarchy works properly
+
+        generated_lines = []
+
+        # Link to the source on GitHub
+        repo_url_template = self.env.config.illiterate_repo_url_template
+        if repo_url_template is not None:
+            repo_url = repo_url_template.format(rel_filename=rel_filename)
+            generated_lines.append(
+                PREAMBLE.format(
+                    name=Path(rel_filename).name,
+                    url=repo_url,
+                )
+            )
+
+        # Paragraph break
+        generated_lines.append("")
+
+        for fragment in fragments:
+            if fragment.kind == "prose":
+                generated_lines.extend([line.content for line in fragment.lines])
+                generated_lines.append("")
+            elif fragment.kind == "code":
+                line_min = fragment.lines[0].lineno
+                line_max = fragment.lines[-1].lineno
+                lines = [
+                    f".. literalinclude:: {self.arguments[0]}",
+                    f"   :language: {language}",
+                    "   :linenos:",
+                    "   :lineno-match:",
+                    f"   :lines: {line_min}-{line_max}",
+                    "",
+                ]
+                generated_lines.extend(lines)
+            else:
+                raise RuntimeError("Unknown fragment kind")
+
+        parsed = docutils.nodes.Element()
+        nested_parse_with_titles(
+            self.state,
+            StringList(generated_lines, source=""),
+            parsed,
+        )
+        return parsed.children
+
+
+def setup(app) -> None:
+    app.add_config_value(
+        "illiterate_repo_url_template",
+        default=None,
+        rebuild="html",
+        types=str,
+    )
+    app.add_directive("recipe", RecipeDirective)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

docs/source/ext/illiterate_sphinx/pyproject.toml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[project]
+name = "illiterate-sphinx"
+description = "A Sphinx directive for not-quite-literate programming."
+authors = [{name = "Apache Arrow Developers", email = "dev@arrow.apache.org"}]
+license = {text = "Apache-2.0"}
+readme = "README.md"
+requires-python = ">=3.9"
+dependencies = ["sphinx"]
+version = "0.1.0"
+
+[project.urls]
+homepage = "https://arrow.apache.org/adbc/"
+repository = "https://github.com/apache/arrow-adbc"
+
+[build-system]
+requires = ["setuptools >= 61.0.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+xfail_strict = true
+
+[tool.setuptools]
+packages = ["illiterate_sphinx"]
+py-modules = ["illiterate_sphinx"]