feat: add Rockcraft-specific Poetry plugin (#717)

The plugin has the same behavior and restrictions as the Python plugin with
regards to base-dependent behavior, symlink handling, sitecustomize, etc.

Therefore, this common behavior is extracted into a new "python_common"
module, used by both plugins. This approach is also taken for the reference
docs - the requirement of staging a Python interpreter is the same for both
plugins.

Fixes #701

.github/workflows/tests.yaml

@@ -81,7 +81,7 @@ jobs:
           sudo apt-get update
           echo "::endgroup::"
           echo "::group::apt-get install..."
-          sudo apt-get install -y python3 python3-dev libapt-pkg-dev libyaml-dev umoci
+          sudo apt-get install -y python3 python3-dev python3-poetry libapt-pkg-dev libyaml-dev umoci
           echo "::endgroup::"
           echo "::group::pip install"
           python -m pip install -U wheel setuptools pip

docs/conf.py

@@ -85,6 +85,7 @@
     "common/craft-parts/reference/parts_steps.rst",
     "common/craft-parts/reference/step_execution_environment.rst",
     "common/craft-parts/reference/step_output_directories.rst",
+    "common/craft-parts/reference/plugins/poetry_plugin.rst",
     "common/craft-parts/reference/plugins/python_plugin.rst",
     # Extra non-craft-parts exclusions can be added after this comment
 ]

docs/reference/plugins.rst

@@ -21,7 +21,7 @@ Rockcraft.
    /common/craft-parts/reference/plugins/meson_plugin
    /common/craft-parts/reference/plugins/nil_plugin
    /common/craft-parts/reference/plugins/npm_plugin
-   /common/craft-parts/reference/plugins/poetry_plugin
+   plugins/poetry_plugin
    plugins/python_plugin
    /common/craft-parts/reference/plugins/qmake_plugin
    /common/craft-parts/reference/plugins/rust_plugin

docs/reference/plugins/_python_common.rst

@@ -0,0 +1,28 @@
+
+Dependencies
+------------
+
+Since none of the bases that are available for rocks contain a default Python
+installation, including a Python interpreter in Rockcraft projects is mandatory.
+Both the ``python`` and the ``poetry`` plugins also require the ``venv`` module
+to create the virtual environment where Python packages are installed at build
+time.
+
+The easiest way to do this is to include the ``python3-venv`` package in the
+``stage-packages`` of the part that uses the Python-based plugin. This will pull
+in the default Python interpreter for the ``build-base``, like Python 3.10 for
+Ubuntu 22.04. However, other versions can be used by explicitly declaring them -
+here's an example that uses ``python3.12-venv`` from the Deadsnakes ppa:
+
+.. code-block:: yaml
+
+   package-repositories:
+     - type: apt
+       ppa: deadsnakes/ppa
+       priority: always
+
+   parts:
+     my-part:
+       plugin: <python or poetry>
+       source: .
+       stage-packages: [python3.12-venv]

docs/reference/plugins/poetry_plugin.rst

@@ -0,0 +1,8 @@
+
+.. include:: /common/craft-parts/reference/plugins/poetry_plugin.rst
+   :end-before: .. _poetry-details-begin:
+
+.. include:: _python_common.rst
+
+.. include:: /common/craft-parts/reference/plugins/poetry_plugin.rst
+   :start-after: .. _poetry-details-end:

docs/reference/plugins/python_plugin.rst

@@ -2,32 +2,7 @@
 .. include:: /common/craft-parts/reference/plugins/python_plugin.rst
    :end-before: .. _python-details-begin:
 
-Dependencies
-------------
-
-Since none of the bases that are available for rocks contain a default Python
-installation, including a Python interpreter in Rockcraft projects is mandatory.
-The plugin also requires the ``venv`` module to create the virtual environment
-where Python packages are installed at build time.
-
-The easiest way to do this is to include the ``python3-venv`` package in the
-``stage-packages`` of the part that uses the Python plugin. This will pull in
-the default Python interpreter for the ``build-base``, like Python 3.10 for
-Ubuntu 22.04. However, other versions can be used by explicitly declaring them -
-here's an example that uses ``python3.12-venv`` from the Deadsnakes ppa:
-
-.. code-block:: yaml
-
-   package-repositories:
-     - type: apt
-       ppa: deadsnakes/ppa
-       priority: always
-
-   parts:
-     my-part:
-       plugin: python
-       source: .
-       stage-packages: [python3.12-venv]
+.. include:: _python_common.rst
 
 .. include:: /common/craft-parts/reference/plugins/python_plugin.rst
    :start-after: .. _python-details-end:

rockcraft/plugins/poetry_plugin.py

@@ -0,0 +1,47 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2024 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""The Rockcraft Poetry plugin."""
+
+
+from craft_parts.plugins import poetry_plugin
+from overrides import override  # type: ignore[reportUnknownVariableType]
+
+from rockcraft.plugins import python_common
+
+
+class PoetryPlugin(poetry_plugin.PoetryPlugin):
+    """A Poetry plugin for Rockcraft."""
+
+    @override
+    def _should_remove_symlinks(self) -> bool:
+        """Overridden because for ubuntu bases we must always remove the symlinks."""
+        return python_common.should_remove_symlinks(self._part_info)
+
+    @override
+    def _get_system_python_interpreter(self) -> str | None:
+        """Overridden because Python must always be provided by the parts."""
+        return None
+
+    @override
+    def _get_script_interpreter(self) -> str:
+        """Overridden because Python is always available in /bin."""
+        return python_common.get_script_interpreter()
+
+    @override
+    def get_build_commands(self) -> list[str]:
+        """Overridden to add a sitecustomize.py."""
+        return python_common.wrap_build_commands(super().get_build_commands())

rockcraft/plugins/python_common.py

@@ -0,0 +1,131 @@
+# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
+#
+# Copyright 2024 Canonical Ltd.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Common functionality for Python-based plugins.
+
+This functionality extends Craft-parts' vanilla Python plugin to properly
+set the Python interpreter according to the rock's base. Specifically:
+
+- If the base is ubuntu, the venv-created symlinks in bin/ are removed
+altogether. This is because of the usrmerge; when the layer is added on
+top of the base ubuntu layer bin/ becomes a symlink to usr/bin/, so there
+already is a usable Python binary in bin/.
+- Since no base (bare or any ubuntu) provides Python by default, the base
+interpreter must always be provided by one of the parts. The easiest way
+to accomplish this is to add "python3-venv" as a stage-package.
+- The shebang in console scripts is hardcoded to "#!/bin/python3". In fact,
+every use of Python in the resulting image should be via /bin/python3.
+"""
+
+
+from textwrap import dedent
+
+import craft_parts
+
+# Template for the sitecustomize module that we'll add to the payload so that
+# the pip-installed packages are found regardless of how the interpreter is
+# called.
+SITECUSTOMIZE_TEMPLATE = dedent(
+    """
+    # sitecustomize added by Rockcraft.
+    import site
+    import sys
+
+    major, minor = sys.version_info.major, sys.version_info.minor
+    site_dir = f"/lib/python{major}.{minor}/site-packages"
+    dist_dir = "/usr/lib/python3/dist-packages"
+
+    # Add the directory that contains the venv-installed packages.
+    site.addsitedir(site_dir)
+
+    if dist_dir in sys.path:
+        # Make sure that this site-packages dir comes *before* the base-provided
+        # dist-packages dir in sys.path.
+        path = sys.path
+        site_index = path.index(site_dir)
+        dist_index = path.index(dist_dir)
+
+        if dist_index < site_index:
+            path[dist_index], path[site_index] = path[site_index], path[dist_index]
+
+    EOF
+    """
+).strip()
+
+
+def should_remove_symlinks(info: craft_parts.PartInfo) -> bool:
+    """Whether a given Python build should remove the python* venv symlinks.
+
+    :param info: the info for the Python-based part.
+    """
+    return bool(info.base != "bare")
+
+
+def get_script_interpreter() -> str:
+    """Python is always available in /bin."""
+    return "#!/bin/${PARTS_PYTHON_INTERPRETER}"
+
+
+def wrap_build_commands(parts_commands: list[str]) -> list[str]:
+    """Wrap the craft-parts build-commands with Rockraft specific code."""
+    commands: list[str] = []
+
+    # Detect whether PARTS_PYTHON_INTERPRETER is a full path (not supported)
+    commands.append(
+        dedent(
+            """
+            # Detect whether PARTS_PYTHON_INTERPRETER is an absolute path
+            if [[ "${PARTS_PYTHON_INTERPRETER}" = /* ]]; then
+                echo "Absolute paths in \"PARTS_PYTHON_INTERPRETER\" are not allowed: ${PARTS_PYTHON_INTERPRETER}"
+                exit 1
+            fi
+            """
+        )
+    )
+
+    commands.extend(parts_commands)
+
+    # Add a "sitecustomize.py" module to handle the very common case of the
+    # rock's interpreter being called as "python3"; in this case, because of
+    # the default $PATH, "/usr/bin/python3" ends up being called and that is
+    # *not* the venv-aware executable. This sitecustomize adds the location
+    # of the pip-installed packages.
+    commands.append(
+        dedent(
+            """
+            # Add a sitecustomize.py to import our venv-generated location
+            py_version=$(basename $payload_python)
+            py_dir=${CRAFT_PART_INSTALL}/usr/lib/${py_version}/
+            mkdir -p ${py_dir}
+            cat << EOF > ${py_dir}/sitecustomize.py
+            """
+        )
+    )
+    commands.append(SITECUSTOMIZE_TEMPLATE)
+
+    # Remove the pyvenv.cfg file that "marks" the virtual environment, because
+    # it's not necessary in the presence of the sitecustomize module and this
+    # way we get consistent behavior no matter how the interpreter is called.
+    commands.append(
+        dedent(
+            """
+            # Remove pyvenv.cfg file in favor of sitecustomize.py
+            rm ${CRAFT_PART_INSTALL}/pyvenv.cfg
+            """
+        )
+    )
+
+    return commands