feat(doc): introduce custom sphinx extension

.gitignore

@@ -2,12 +2,10 @@ __pycache__
 # do not track doc files that are automatically build
 doc/build
 doc/source/*.rst
+doc/source/*.md
 
 # but keep index.rst
 !doc/source/index.rst
-!doc/source/intro.rst
-!doc/source/tutorials.rst
-!doc/source/architecture.rst
 
 # C extensions
 *.so

CONTRIBUTING.md

@@ -56,7 +56,7 @@ The code checks are conducted with [Pylint](https://pylint.org/),
 Compliance with [Google style docstrings](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
 is checked with [pydocstyle](https://github.com/PyCQA/pydocstyle).
 Complete and meaningful docstrings are required as they are used to generate the
-[documentation](#reading-and-writing-documentation).
+[documentation](#book-documentation).
 
 #### Commit messages
 Please provide meaningful commit messages based on the

dev-requirements.in

@@ -20,3 +20,4 @@ commitizen>=3.12.0
 pydocstyle>=6.3.0
 docformatter>=1.5.1
 yamllint>=1.19.0
+myst-parser

dev-requirements.txt

@@ -62,6 +62,7 @@ docformatter==1.7.5
     # via -r dev-requirements.in
 docutils==0.21.2
     # via
+    #   myst-parser
     #   nbsphinx
     #   pydata-sphinx-theme
     #   sphinx
@@ -87,6 +88,7 @@ jinja2==3.1.4
     # via
     #   -c requirements.txt
     #   commitizen
+    #   myst-parser
     #   nbconvert
     #   nbsphinx
     #   sphinx
@@ -108,15 +110,28 @@ liccheck==0.9.2
     # via -r dev-requirements.in
 licenseheaders==0.8.8
     # via -r dev-requirements.in
+markdown-it-py==3.0.0
+    # via
+    #   -c requirements.txt
+    #   mdit-py-plugins
+    #   myst-parser
 markupsafe==3.0.2
     # via
     #   -c requirements.txt
     #   jinja2
     #   nbconvert
 mccabe==0.7.0
     # via pylint
+mdit-py-plugins==0.4.2
+    # via myst-parser
+mdurl==0.1.2
+    # via
+    #   -c requirements.txt
+    #   markdown-it-py
 mistune==3.0.2
     # via nbconvert
+myst-parser==4.0.0
+    # via -r dev-requirements.in
 nbclient==0.10.0
     # via nbconvert
 nbconvert==7.16.4
@@ -190,6 +205,7 @@ pyyaml==6.0.2
     # via
     #   -c requirements.txt
     #   commitizen
+    #   myst-parser
     #   pre-commit
     #   yamllint
 pyzmq==26.2.0
@@ -230,6 +246,7 @@ soupsieve==2.6
 sphinx==8.1.3
     # via
     #   -r dev-requirements.in
+    #   myst-parser
     #   nbsphinx
     #   pydata-sphinx-theme
 sphinxcontrib-applehelp==2.0.0

doc/source/_ext/create_documentation_files.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2025, QUEENS contributors.
+#
+# This file is part of QUEENS.
+#
+# QUEENS is free software: you can redistribute it and/or modify it under the terms of the GNU
+# Lesser General Public License as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version. QUEENS is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You
+# should have received a copy of the GNU Lesser General Public License along with QUEENS. If not,
+# see <https://www.gnu.org/licenses/>.
+#
+"""Documentation creation utils."""
+
+import pydoc
+import re
+import sys
+from pathlib import Path
+
+import requests
+
+from queens.utils.injector import inject
+from queens.utils.path_utils import relative_path_from_queens
+
+sys.path.insert(1, str(relative_path_from_queens("test_utils").resolve()))
+from get_queens_example_from_readme import (  # pylint: disable=import-error, wrong-import-position,wrong-import-order
+    extract_from_markdown_file_by_marker,
+    get_queens_example_from_readme,
+)
+
+
+def get_template_path_by_name(template_name):
+    """Get template path by name from the template folder.
+
+    Args:
+        template_name (str): Temple name in the template folder.
+
+    Returns:
+        pathlib.Path: Path to template
+    """
+    template_path = (Path(__file__).parent / "templates") / template_name
+    return template_path
+
+
+def relative_to_doc_source(relative_path):
+    """Relative path from documentation source.
+
+    Args:
+        relative_path (str): path relative to `doc/source/`
+
+    Returns:
+        pathlib.Path: Path relative from documentation
+    """
+    return relative_path_from_queens("doc/source/" + relative_path)
+
+
+def create_tutorial_from_readme():
+    """Create tutorial from readme."""
+    example = get_queens_example_from_readme(".")
+    tutorial_template = get_template_path_by_name("tutorials.rst.j2")
+    tutorial = relative_to_doc_source("tutorials.rst")
+
+    inject({"example_text": example.replace("\n", "\n   ")}, tutorial_template, tutorial)
+
+
+def remove_markdown_emojis(md_text):
+    """Remove emojis from markdown text.
+
+    Args:
+        md_text (str): Markdown text
+
+    Returns:
+        str: Cleaned text
+    """
+    # Define a regex pattern for matching markdown-style emojis
+    emoji_pattern = re.compile(r":\w+:")
+
+    for emoji in emoji_pattern.findall(md_text):
+
+        # Remove markdown emojis from the text
+        md_text = md_text.replace(emoji, "")
+
+        # Replace emojis in reference to other sections
+        md_text = md_text.replace(emoji[1:-1], "")
+
+    return md_text
+
+
+def prepend_relative_links(md_text, base_url):
+    """Prepend url for relative links.
+
+    Args:
+        md_text (str): Text to check
+        base_url (str): Base URL to add
+
+    Returns:
+        str: Prepended markdown text
+    """
+    md_link_regex = "\\[([^]]+)]\\(\\s*(.*)\\s*\\)"
+    for match in re.findall(md_link_regex, md_text):
+        _, link = match
+
+        # For local reference that started with an emoji
+        if link.strip().startswith("#-"):
+            new_link = "#" + link[2:].strip().lower()
+            md_text = md_text.replace(f"({link})", f"({new_link})")
+
+        # No http links or proper references within the file references
+        if not link.strip().startswith("http") and not link.strip().startswith("#"):
+            md_text = md_text.replace(f"({link})", f"({base_url}/{link.strip()})")
+
+    return md_text
+
+
+def clean_markdown(md_text):
+    """Clean markdown.
+
+    Removes emojis and prepends links.
+
+    Args:
+        md_text (str): Original markdown text.
+
+    Returns:
+        str: Markdown text
+    """
+    md_text = remove_markdown_emojis(md_text)
+    md_text = prepend_relative_links(md_text, "https://www.github.com/queens-py/queens/blob/main")
+    return md_text
+
+
+def clean_markdown_file(md_path, new_path):
+    """Load markdown and escape relative links and emojis.
+
+    Args:
+        md_path (pathlib.Path, str): Path to an existing markdown file
+        new_path (pathlib.Path, str): Path for the cleaned file
+
+    Returns:
+        str: file name of the new markdown file
+    """
+    md_text = clean_markdown(relative_path_from_queens(md_path).read_text())
+    new_path = Path(new_path)
+    new_path.write_text(md_text, encoding="utf-8")
+    return new_path.name
+
+
+def create_development():
+    """Create development page."""
+    development_template = get_template_path_by_name("development.rst.j2")
+    development_path = relative_to_doc_source("development.rst")
+
+    md_paths = []
+    md_paths.append(
+        clean_markdown_file(
+            relative_path_from_queens("CONTRIBUTING.md"), relative_to_doc_source("contributing.md")
+        )
+    )
+    md_paths.append(
+        clean_markdown_file(
+            relative_path_from_queens("tests/README.md"), relative_to_doc_source("testing.md")
+        )
+    )
+    inject({"md_paths": md_paths}, development_template, development_path)
+
+
+def create_intro():
+    """Generate landing page."""
+    intro_template = get_template_path_by_name("intro.md.j2")
+    into_path = relative_to_doc_source("intro.md")
+
+    def extract_from_markdown_by_marker(marker_name, md_path):
+        return clean_markdown(extract_from_markdown_file_by_marker(marker_name, md_path))
+
+    inject(
+        {
+            "readme_path": relative_path_from_queens("README.md"),
+            "contributing_path": relative_path_from_queens("CONTRIBUTING.md"),
+            "extract_from_markdown_by_marker": extract_from_markdown_by_marker,
+        },
+        intro_template,
+        into_path,
+    )
+
+
+def create_overview():
+    """Create overview of the QUEENS package."""
+    overview_template = get_template_path_by_name("overview.rst.j2")
+    overview_path = relative_to_doc_source("overview.rst")
+
+    queens_base_path = relative_path_from_queens("queens")
+
+    def get_module_description(python_file):
+        """Get module description.
+
+        Args:
+            python_file (pathlib.Path): Path to python file.
+
+        Returns:
+            str: Module description.
+        """
+        module_documentation = pydoc.importfile(str(python_file)).__doc__.split("\n\n")
+        return "\n\n".join([m.replace("\n", " ") for m in module_documentation[1:]])
+
+    modules = []
+    for path in sorted(queens_base_path.iterdir()):
+        if path.name.startswith("__") or not path.is_dir():
+            continue
+
+        description = get_module_description(path / "__init__.py")
+        name = path.stem
+
+        modules.append(
+            {
+                "name": name.replace("_", " ").title(),
+                "module": "queens." + name,
+                "description": description,
+            }
+        )
+
+    inject({"modules": modules, "len": len}, overview_template, overview_path)
+
+
+def download_images():
+    """Download images."""
+
+    def download_file_from_url(url, file_name):
+        """Download file from an url."""
+        url_request = requests.get(url, timeout=10)
+        with open(file_name, "wb") as f:
+            f.write(url_request.content)
+
+    download_file_from_url(
+        "https://raw.githubusercontent.com/queens-py/queens-design/main/logo/queens_logo_night.svg",
+        relative_to_doc_source("images/queens_logo_night.svg"),
+    )
+    download_file_from_url(
+        "https://raw.githubusercontent.com/queens-py/queens-design/main/logo/queens_logo_day.svg",
+        relative_to_doc_source("images/queens_logo_day.svg"),
+    )
+
+
+def main():
+    """Create all the rst files."""
+    create_intro()
+    create_tutorial_from_readme()
+    create_development()
+    create_overview()

doc/source/_ext/queens_sphinx_extension.py

@@ -0,0 +1,38 @@
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2025, QUEENS contributors.
+#
+# This file is part of QUEENS.
+#
+# QUEENS is free software: you can redistribute it and/or modify it under the terms of the GNU
+# Lesser General Public License as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version. QUEENS is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You
+# should have received a copy of the GNU Lesser General Public License along with QUEENS. If not,
+# see <https://www.gnu.org/licenses/>.
+#
+"""This is a custom extension to create files for sphinx using python."""
+
+import os
+import sys
+
+from sphinx.application import Sphinx
+
+sys.path.insert(0, os.path.abspath("."))
+
+import create_documentation_files  # pylint: disable=wrong-import-position
+
+
+def run_custom_code(app: Sphinx):  # pylint: disable=unused-argument
+    """Run the custom code."""
+    create_documentation_files.main()
+
+
+def setup(app: Sphinx):  # pylint: disable=unused-argument
+    """Setup up sphinx app."""
+    app.connect("builder-inited", run_custom_code)
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+    }

doc/source/_ext/templates/development.rst.j2

@@ -0,0 +1,11 @@
+Development
+===========
+
+Thanks for your interest in developing QUEENS!
+
+.. toctree::
+   :maxdepth: 1
+
+
+{% for md_path in md_paths %}
+   {{ md_path }}{% endfor %}