✨ Add sphinxcontrib-tree-sitter

Author: Freed-Wu

README.md

@@ -12,5 +12,40 @@ This project provides:
     [texcat](https://texrocks.readthedocs.io/en/latest/topics/texcat.md.html)
   - [LaTeX](https://www.latex-project.org/): also
     [texcat](https://texrocks.readthedocs.io/en/latest/topics/texcat.md.html)
+  - [jekyll](https://github.com/jekyll/jekyll): TODO
   - [pandoc](http://github.com/pandoc/pandoc): TODO
   - [typst](https://github.com/typst/typst): TODO
+
+## Bench
+
+<!-- markdownlint-disable MD013 -->
+
+```bash
+$ hyperfine -Nw10 'python -m sphinxcontrib.tree_sitter pyproject.toml' 'texcat pyproject.toml' 'tree-sitter highlight pyproject.toml' 'pygmentize pyproject.toml' 'bat pyproject.toml'
+Benchmark 1: python -m sphinxcontrib.tree_sitter pyproject.toml
+  Time (mean ± σ):     367.2 ms ±  25.7 ms    [User: 307.1 ms, System: 51.4 ms]
+  Range (min … max):   345.6 ms … 414.5 ms    10 runs
+
+Benchmark 2: texcat pyproject.toml
+  Time (mean ± σ):      70.3 ms ±   3.0 ms    [User: 61.0 ms, System: 8.3 ms]
+  Range (min … max):    67.2 ms …  81.6 ms    42 runs
+
+Benchmark 3: tree-sitter highlight pyproject.toml
+  Time (mean ± σ):       5.8 ms ±   0.3 ms    [User: 2.6 ms, System: 2.9 ms]
+  Range (min … max):     5.2 ms …   8.1 ms    531 runs
+
+Benchmark 4: pygmentize pyproject.toml
+  Time (mean ± σ):     250.5 ms ±  13.3 ms    [User: 221.1 ms, System: 24.3 ms]
+  Range (min … max):   238.9 ms … 274.7 ms    10 runs
+
+Benchmark 5: bat pyproject.toml
+  Time (mean ± σ):      15.0 ms ±   3.4 ms    [User: 11.8 ms, System: 6.6 ms]
+  Range (min … max):    11.7 ms …  29.4 ms    252 runs
+
+Summary
+  tree-sitter highlight pyproject.toml ran
+    2.60 ± 0.60 times faster than bat pyproject.toml
+   12.20 ± 0.86 times faster than texcat pyproject.toml
+   43.47 ± 3.35 times faster than pygmentize pyproject.toml
+   63.73 ± 5.71 times faster than python -m sphinxcontrib.tree_sitter pyproject.toml
+```

crates/py-tree-sitter-highlight/README.md

@@ -8,7 +8,7 @@ from tree_sitter_highlight import highlight, search_parsers
 
 parsers = search_parsers(
     tree_sitter_python,
-    # same as:
+    # NOTE: same as
     # python=tree_sitter_python,
     sh = tree_sitter_bash,
 )

crates/py-tree-sitter-highlight/pyproject.toml

@@ -20,14 +20,6 @@ dev = [
   "pytest-cov>=7",
   "tree-sitter-python>=0.25.0",
 ]
-docs = [
-  "myst-parser>=4",
-  "tree-sitter-bash>=0.25",
-  "tree-sitter-comment>=0.3.0",
-  "tree-sitter-lua>=0.5.0",
-  "tree-sitter-python>=0.25.0",
-  "tree-sitter-toml>=0.7.0",
-]
 
 [build-system]
 requires = ["maturin>=1.0,<2.0"]

crates/py-tree-sitter-highlight/tree_sitter_highlight/tree_sitter_highlight.pyi

@@ -1,13 +1,7 @@
 from __future__ import annotations
 
-import sys
 from typing import Any, Literal, TypedDict
 
-if sys.version_info >= (3, 11):
-    from typing import NotRequired
-else:
-    from typing_extensions import NotRequired
-
 # An imported ``tree_sitter_*`` grammar package module object, e.g.
 # ``import tree_sitter_bash``. The package exposes a ``language()`` function
 # returning a pointer (as an ``int``) plus on-disk query files; this binding
@@ -20,7 +14,6 @@ Style = Literal["classes", "inline", "minimal"]
 
 __all__ = ["search_parsers", "highlight"]
 
-
 class ParserEntry(TypedDict):
     """The per-language entry stored in the ``parsers`` mapping."""
 
@@ -36,14 +29,12 @@ class ParserEntry(TypedDict):
     locals: str
     """The text of the language's ``locals.scm`` query (may be empty)."""
 
-
 # Mapping of language name -> ParserEntry.
 Parsers = dict[str, ParserEntry]
 
 # A theme mapping highlight scope names to ``{color, bold, italic, ...}``.
 Theme = dict[str, Any]
 
-
 def search_parsers(
     *modules: GrammarModule,
     **named_modules: GrammarModule,
@@ -62,7 +53,6 @@ def search_parsers(
     ``LOCALS_QUERY`` module attributes).
     """
 
-
 def highlight(
     *,
     source: str | None = None,

docs/conf.py

@@ -25,7 +25,10 @@
 copyright = "2026-" + str(datetime.now().year)
 
 PROJECT_FILE = os.path.join(
-    os.path.dirname(os.path.dirname(__file__)), "pyproject.toml"
+    os.path.dirname(os.path.dirname(__file__)),
+    "packages",
+    "sphinxcontrib-tree-sitter",
+    "pyproject.toml",
 )
 
 with open(PROJECT_FILE, "rb") as f:

packages/sphinxcontrib-tree-sitter/README.md

@@ -0,0 +1,31 @@
+# sphinxcontrib-tree-sitter
+
+Use tree-sitter to highlight code blocks in Sphinx.
+
+## Usage
+
+`docs/conf.py`:
+
+```python
+extensions = [
+    "sphinxcontrib.tree_sitter",
+]
+```
+
+`pyproject.toml`:
+
+```toml
+# ...
+[project.optional-dependencies]
+dev = [
+  "sphinx",
+  "tree-sitter-XXX",
+]
+```
+
+```bash
+uv sync
+source .venv/bin/activate
+sphinx-build docs _readthedocs/html
+xdg-open _readthedocs/html/index.html
+```

packages/sphinxcontrib-tree-sitter/lux.toml

@@ -0,0 +1,14 @@
+package = "bench"
+version = "0.0.1"
+lua = ">=5.1"
+
+[description]
+summary = ""
+maintainer = "Wu"
+labels = [ "" ]
+
+[test_dependencies]
+texcat = ">=0.0.1"
+
+[build]
+type = "builtin"

packages/sphinxcontrib-tree-sitter/pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "sphinxcontrib-tree-sitter"
+version = "0.0.1"
+description = "Use tree-sitter to highlight code blocks in Sphinx"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "GPL-3.0-or-later"
+authors = [{ name = "Wu, Zhenyu", email = "wuzhenyu@ustc.edu" }]
+dependencies = [
+  "identify>=2",
+  "pygments>=2",
+  "tree-sitter-highlight",
+]
+
+[project.urls]
+homepage = "https://github.com/sphinx-contrib/tree-sitter-highlight"
+
+[project.optional-dependencies]
+dev = [
+  "myst-parser>=4",
+  "tree-sitter-bash>=0.25",
+  "tree-sitter-comment>=0.3.0",
+  "tree-sitter-lua>=0.5.0",
+  "tree-sitter-python>=0.25.0",
+  "tree-sitter-toml>=0.7.0",
+]
+
+[build-system]
+requires = ["uv_build<0.12.0"]
+build-backend = "uv_build"
+
+[tool.uv.build-backend]
+module-name = "sphinxcontrib.tree_sitter"
+
+[tool.uv.sources]
+tree-sitter-highlight = { workspace = true }

packages/sphinxcontrib-tree-sitter/src/sphinxcontrib/tree_sitter/__init__.py

@@ -0,0 +1,21 @@
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+__version__ = "0.0.1"
+
+
+def setup(_: "Sphinx") -> dict[str, str | bool]:
+    from sphinx.highlighting import PygmentsBridge
+
+    from .highlighting import TreeSitterBridge
+
+    PygmentsBridge.highlight_block = TreeSitterBridge.highlight_block  # ty:ignore[invalid-assignment]
+    PygmentsBridge.get_stylesheet = TreeSitterBridge.get_stylesheet  # ty:ignore[invalid-assignment]
+
+    return {
+        "version": __version__,
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

packages/sphinxcontrib-tree-sitter/src/sphinxcontrib/tree_sitter/__main__.py

@@ -0,0 +1,109 @@
+r"""This module can be called by
+`python -m <https://docs.python.org/3/library/__main__.html>`_.
+"""
+
+from argparse import ArgumentParser
+
+from pygments.styles import STYLE_MAP, get_style_by_name
+from tree_sitter_highlight import highlight
+
+from . import __version__
+from .highlighting import TreeSitterBridge
+
+
+def get_parser(parsers) -> ArgumentParser:
+    r"""Get a parser for unit test."""
+    parser = ArgumentParser()
+    parser.add_argument("--version", version=__version__, action="version")
+
+    parser.add_argument("files", nargs="*", help="file name")
+    parser.add_argument(
+        "--language",
+        choices=parsers,
+        help="set language",
+    )
+    parser.add_argument(
+        "--theme",
+        choices=STYLE_MAP,
+        help="set theme",
+    )
+    parser.add_argument(
+        "--format",
+        default="terminal",
+        choices=["latex", "html", "terminal"],
+        help="output format (default: terminal)",
+    )
+    parser.add_argument(
+        "--layout",
+        default="document",
+        choices=["fragment", "document", "line-numbers"],
+        help="layout (default: document)",
+    )
+    parser.add_argument(
+        "--style",
+        default="classes",
+        choices=["minimal", "classes", "inline"],
+        help="style (default: classes)",
+    )
+    parser.add_argument(
+        "--prefix", default="TS", help="command prefix for TeX (default: TS)"
+    )
+    parser.add_argument(
+        "--math-escape",
+        nargs="*",
+        default=[],
+        help="the scope to escape $math TeX code$",
+    )
+    parser.add_argument(
+        "--injections",
+        nargs="*",
+        default=[],
+        help="injection languages, all means all",
+    )
+    return parser
+
+
+def main() -> None:
+    r"""Parse arguments and provide shell completions."""
+    parsers = TreeSitterBridge.get_parsers()
+    parser = get_parser(parsers)
+    args = parser.parse_args()
+
+    from identify import identify
+
+    theme = None
+    if args.theme:
+        style = get_style_by_name(args.theme)
+        theme = TreeSitterBridge.get_theme(style)
+    for file in args.files:
+        language = args.language
+        if language is None:
+            tags = identify.tags_from_path(file)
+            for tag in tags:
+                if tag in parsers:
+                    language = tag
+                    break
+            if language is None:
+                language = "text"
+        if language == "text":
+            with open(file) as f:
+                text = f.read()
+            print(text)
+        elif language in parsers:
+            hlsource = highlight(
+                file=file,
+                language=language,
+                parsers=parsers,
+                theme=theme,
+                layout=args.layout,
+                style=args.style,
+                prefix=args.prefix,
+                math_escape=args.math_escape,
+            )
+            print(hlsource)
+        else:
+            print(f"No parser found for {language}")
+
+
+if __name__ == "__main__":
+    main()