feat(doc): introduce custom sphinx extension

.gitignore

@@ -2,11 +2,11 @@ __pycache__
 # do not track doc files that are automatically build
 doc/build
 doc/source/*.rst
+doc/source/*.md
 
 # but keep index.rst
 !doc/source/index.rst
 !doc/source/intro.rst
-!doc/source/tutorials.rst
 !doc/source/architecture.rst
 
 # C extensions

dev-requirements.in

@@ -20,3 +20,4 @@ commitizen>=3.12.0
 pydocstyle>=6.3.0
 docformatter>=1.5.1
 yamllint>=1.19.0
+myst-parser

dev-requirements.txt

@@ -62,6 +62,7 @@ docformatter==1.7.5
     # via -r dev-requirements.in
 docutils==0.21.2
     # via
+    #   myst-parser
     #   nbsphinx
     #   pydata-sphinx-theme
     #   sphinx
@@ -87,6 +88,7 @@ jinja2==3.1.4
     # via
     #   -c requirements.txt
     #   commitizen
+    #   myst-parser
     #   nbconvert
     #   nbsphinx
     #   sphinx
@@ -108,15 +110,28 @@ liccheck==0.9.2
     # via -r dev-requirements.in
 licenseheaders==0.8.8
     # via -r dev-requirements.in
+markdown-it-py==3.0.0
+    # via
+    #   -c requirements.txt
+    #   mdit-py-plugins
+    #   myst-parser
 markupsafe==3.0.2
     # via
     #   -c requirements.txt
     #   jinja2
     #   nbconvert
 mccabe==0.7.0
     # via pylint
+mdit-py-plugins==0.4.2
+    # via myst-parser
+mdurl==0.1.2
+    # via
+    #   -c requirements.txt
+    #   markdown-it-py
 mistune==3.0.2
     # via nbconvert
+myst-parser==4.0.0
+    # via -r dev-requirements.in
 nbclient==0.10.0
     # via nbconvert
 nbconvert==7.16.4
@@ -190,6 +205,7 @@ pyyaml==6.0.2
     # via
     #   -c requirements.txt
     #   commitizen
+    #   myst-parser
     #   pre-commit
     #   yamllint
 pyzmq==26.2.0
@@ -230,6 +246,7 @@ soupsieve==2.6
 sphinx==8.1.3
     # via
     #   -r dev-requirements.in
+    #   myst-parser
     #   nbsphinx
     #   pydata-sphinx-theme
 sphinxcontrib-applehelp==2.0.0

doc/source/_ext/create_documentation_files.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2025, QUEENS contributors.
+#
+# This file is part of QUEENS.
+#
+# QUEENS is free software: you can redistribute it and/or modify it under the terms of the GNU
+# Lesser General Public License as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version. QUEENS is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You
+# should have received a copy of the GNU Lesser General Public License along with QUEENS. If not,
+# see <https://www.gnu.org/licenses/>.
+#
+"""Documentation creation utils."""
+
+import pydoc
+import re
+import sys
+from pathlib import Path
+
+import requests
+
+from queens.utils.injector import inject
+from queens.utils.path_utils import relative_path_from_queens
+
+sys.path.insert(1, str(relative_path_from_queens("test_utils").resolve()))
+from get_queens_example_from_readme import (  # pylint: disable=import-error, wrong-import-position,wrong-import-order
+    extract_from_markdown_file_by_marker,
+    get_queens_example_from_readme,
+)
+
+
+def get_template_path_by_name(template_name):
+    """Get template path by name from the template folder.
+
+    Args:
+        template_name (str): Temple name in the template folder.
+
+    Returns:
+        pathlib.Path: Path to template
+    """
+    template_path = (Path(__file__).parent / "templates") / template_name
+    return template_path
+
+
+def relative_to_doc_source(relative_path):
+    """Relative path from documentation source.
+
+    Args:
+        relative_path (str): path relative to `doc/source/`
+
+    Returns:
+        pathlib.Path: Path relative from documentation
+    """
+    return relative_path_from_queens("doc/source/" + relative_path)
+
+
+def create_tutorial_from_readme():
+    """Create tutorial from readme."""
+    example = get_queens_example_from_readme(".")
+    tutorial_template = get_template_path_by_name("tutorials.rst.j2")
+    tutorial = relative_to_doc_source("tutorials.rst")
+
+    inject({"example_text": example.replace("\n", "\n   ")}, tutorial_template, tutorial)
+
+
+def remove_markdown_emojis(md_text):
+    """Remove emojis from markdown text.
+
+    Args:
+        md_text (str): Markdown text
+
+    Returns:
+        str: Cleaned text
+    """
+    # Define a regex pattern for matching markdown-style emojis
+    emoji_pattern = re.compile(r":\w+:")
+
+    # Remove markdown emojis from the text
+    return emoji_pattern.sub("", md_text)
+
+
+def prepend_relative_links(md_text, base_url):
+    """Prepend url for relative links.
+
+    Args:
+        md_text (str): Text to check
+        base_url (str): Base URL to add
+
+    Returns:
+        str: Prepended markdown text
+    """
+    md_link_regex = "\\[([^]]+)]\\(\\s*(.*)\\s*\\)"
+    for match in re.findall(md_link_regex, md_text):
+        _, link = match
+        if link.strip().startswith("http"):
+            continue
+        md_text = md_text.replace(f"({link})", f"({base_url}/{link.strip()})")
+    return md_text
+
+
+def clean_markdown(md_text):
+    """Clean markdown.
+
+    Removes emojis and prepends links.
+
+    Args:
+        md_text (str): Original markdown text.
+
+    Returns:
+        str: Markdown text
+    """
+    md_text = remove_markdown_emojis(md_text)
+    md_text = prepend_relative_links(md_text, "https://www.github.com/queens-py/queens/blob/main")
+    return md_text
+
+
+def create_markdown_import(md_path, new_path):
+    """Load markdown and escape relative links and emojis."""
+    md_text = clean_markdown(relative_path_from_queens(md_path).read_text())
+    new_path = Path(new_path)
+    new_path.write_text(md_text, encoding="utf-8")
+    return new_path.name
+
+
+def create_development():
+    """Create development page."""
+    development_template = get_template_path_by_name("development.rst.j2")
+    development_path = relative_to_doc_source("development.rst")
+
+    md_paths = []
+    md_paths.append(
+        create_markdown_import(
+            relative_path_from_queens("CONTRIBUTING.md"), relative_to_doc_source("contributing.md")
+        )
+    )
+    md_paths.append(
+        create_markdown_import(
+            relative_path_from_queens("tests/README.md"), relative_to_doc_source("testing.md")
+        )
+    )
+    inject({"md_paths": md_paths}, development_template, development_path)
+
+
+def create_intro():
+    """Generate landing page."""
+    intro_template = get_template_path_by_name("intro.md.j2")
+    into_path = relative_to_doc_source("intro.md")
+
+    def extract_from_markdown_by_marker(marker_name, md_path):
+        return clean_markdown(extract_from_markdown_file_by_marker(marker_name, md_path))
+
+    inject(
+        {
+            "readme_path": relative_path_from_queens("README.md"),
+            "contributing_path": relative_path_from_queens("CONTRIBUTING.md"),
+            "extract_from_markdown_by_marker": extract_from_markdown_by_marker,
+        },
+        intro_template,
+        into_path,
+    )
+
+
+def create_overview():
+    """Create overview of the QUEENS package."""
+    overview_template = get_template_path_by_name("overview.rst.j2")
+    overview_path = relative_to_doc_source("overview.rst")
+
+    queens_base_path = relative_path_from_queens("queens")
+
+    def get_module_description(python_file):
+        """Get module description.
+
+        Args:
+            python_file (pathlib.Path): Path to python file.
+
+        Returns:
+            str: Module description.
+        """
+        module_documentation = pydoc.importfile(str(python_file)).__doc__.split("\n\n")
+        return "\n\n".join([m.replace("\n", " ") for m in module_documentation[1:]])
+
+    modules = []
+    for path in sorted(queens_base_path.iterdir()):
+        if path.name.startswith("__") or not path.is_dir():
+            continue
+
+        description = get_module_description(path / "__init__.py")
+        name = path.stem
+
+        modules.append(
+            {
+                "name": name.replace("_", " ").title(),
+                "module": "queens." + name,
+                "description": description,
+            }
+        )
+
+    inject({"modules": modules, "len": len}, overview_template, overview_path)
+
+
+def download_images():
+    """Download images."""
+
+    def download_file_from_url(url, file_name):
+        """Download file from an url."""
+        url_request = requests.get(url, timeout=10)
+        with open(file_name, "wb") as f:
+            f.write(url_request.content)
+
+    download_file_from_url(
+        "https://raw.githubusercontent.com/queens-py/queens-design/main/logo/queens_logo_night.svg",
+        relative_to_doc_source("images/queens_logo_night.svg"),
+    )
+    download_file_from_url(
+        "https://raw.githubusercontent.com/queens-py/queens-design/main/logo/queens_logo_day.svg",
+        relative_to_doc_source("images/queens_logo_day.svg"),
+    )
+
+
+def main():
+    """Create all the rst files."""
+    create_intro()
+    create_tutorial_from_readme()
+    create_development()
+    create_overview()

doc/source/_ext/queens_sphinx_extension.py

@@ -0,0 +1,41 @@
+#
+# SPDX-License-Identifier: LGPL-3.0-or-later
+# Copyright (c) 2025, QUEENS contributors.
+#
+# This file is part of QUEENS.
+#
+# QUEENS is free software: you can redistribute it and/or modify it under the terms of the GNU
+# Lesser General Public License as published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version. QUEENS is distributed in the hope that it will
+# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You
+# should have received a copy of the GNU Lesser General Public License along with QUEENS. If not,
+# see <https://www.gnu.org/licenses/>.
+#
+"""This is a custom extension to create files for sphinx using python.
+
+Not sure what this exactly does, but at long as it works it's fine.
+"""
+
+import os
+import sys
+
+from sphinx.application import Sphinx
+
+sys.path.insert(0, os.path.abspath("."))
+
+import create_documentation_files  # pylint: disable=wrong-import-position
+
+
+def run_custom_code(app: Sphinx):  # pylint: disable=unused-argument
+    """Run the custom code."""
+    create_documentation_files.main()
+
+
+def setup(app: Sphinx):  # pylint: disable=unused-argument
+    """Setup up sphinx app."""
+    app.connect("builder-inited", run_custom_code)
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+    }

doc/source/_ext/templates/development.rst.j2

@@ -0,0 +1,11 @@
+Development
+===========
+
+Thanks for your interest in developing QUEENS!
+
+.. toctree::
+   :maxdepth: 1
+
+
+{% for md_path in md_paths %}
+   {{ md_path }}{% endfor %}

doc/source/_ext/templates/intro.md.j2

@@ -0,0 +1,33 @@
+# Introduction
+
+```{image} images/queens_logo_day.svg
+:class: only-light
+```
+
+```{image} images/queens_logo_night.svg
+:class: only-dark
+```
+
+{{ extract_from_markdown_by_marker("description", readme_path) }}
+
+## Capabilities
+{{ extract_from_markdown_by_marker("capabilities", readme_path) }}
+
+
+## Installation
+{{ extract_from_markdown_by_marker("prerequisites", readme_path) }}
+
+{{ extract_from_markdown_by_marker("installation", readme_path) }}
+
+For development, install the additional required packages via:
+{{ extract_from_markdown_by_marker("installation_develop", contributing_path) }}
+
+> Note: We recommend using conda/mamba environments and installing performance-critical packages (e.g., numpy, scipy, ...) using `conda install <packagename>.` The reason for this is the choice of BLAS library (linear algebra packages). Conda (depending on the channel) installs numpy and the [mkl](https://www.intel.com/content/www/us/en/developer/articles/technical/intel-mkl-and-third-party-applications-how-to-use-them-together.html) library from Intel, in contrast to pip which defaults back to the linear algebra package installed on the system. According to certain benchmarks ([here](http://markus-beuckelmann.de/blog/boosting-numpy-blas.html) or [here](https://medium.com/analytics-vidhya/why-conda-install-instead-of-pip-install-ba4c6826a0ae)), the mkl library is able to outperform other linear algebra libraries, especially on Intel devices. Particularly for use cases where linear algebra operations dominate the computational costs, the benefit can be huge.
+
+## Citing QUEENS
+You used QUEENS for a publication? Don't forget to cite
+{{ extract_from_markdown_by_marker("citation", readme_path) }}
+and the respective methods papers.
+
+## License
+{{ extract_from_markdown_by_marker("license", readme_path) }}