pants-plugins/uses_services: add support for uses=st2cluster in BUILD…

… metadata (#6279)

CHANGELOG.rst

@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269 #6275
+  #6254 #6258 #6259 #6260 #6269 #6275 #6279
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

contrib/runners/orquesta_runner/tests/integration/BUILD

@@ -5,5 +5,6 @@ __defaults__(
 
 python_tests(
     name="tests",
-    uses=["mongo", "rabbitmq", "redis", "system_user"],
+    uses=["mongo", "rabbitmq", "redis", "st2cluster", "system_user"],
+    tags=["integration", "st2cluster"],
 )

pants-plugins/uses_services/register.py

@@ -21,6 +21,7 @@
     platform_rules,
     rabbitmq_rules,
     redis_rules,
+    st2cluster_rules,
     system_user_rules,
 )
 from uses_services.target_types import UsesServicesField
@@ -36,5 +37,6 @@ def rules():
         *mongo_rules.rules(),
         *rabbitmq_rules.rules(),
         *redis_rules.rules(),
+        *st2cluster_rules.rules(),
         *system_user_rules.rules(),
     ]

pants-plugins/uses_services/scripts/is_st2cluster_running.py

@@ -0,0 +1,52 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import socket
+import sys
+
+from contextlib import closing
+
+
+def _is_st2cluster_running(endpoints: list[tuple[str, str]]) -> bool:
+    """Check for listening ports of st2auth, st2api, and st2stream services.
+
+    This should not import the st2 code as it should be self-contained.
+    """
+    # TODO: Once each service gains a reliable health check endpoint, use that.
+    # https://github.com/StackStorm/st2/issues/4020
+    for host, port in endpoints:
+        # based on https://stackoverflow.com/a/35370008/1134951
+        with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
+            # errno=0 means the connection succeeded
+            if sock.connect_ex((host, int(port))) != 0:
+                # failed to create a connection to the port.
+                return False
+    return True
+
+
+if __name__ == "__main__":
+    args_iter = iter(sys.argv[1:])
+    # Turn the list into 2 tuples (zip with query the same iterator twice for each entry)
+    endpoints = list(zip(args_iter, args_iter))
+    if not endpoints:
+        endpoints = [
+            ("127.0.0.1", "9100"),
+            ("127.0.0.1", "9101"),
+            ("127.0.0.1", "9102"),
+        ]
+
+    is_running = _is_st2cluster_running(endpoints)
+    exit_code = 0 if is_running else 1
+    sys.exit(exit_code)

pants-plugins/uses_services/st2cluster_rules.py

@@ -0,0 +1,191 @@
+# Copyright 2024 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+from dataclasses import dataclass
+from textwrap import dedent
+
+from pants.backend.python.goals.pytest_runner import (
+    PytestPluginSetupRequest,
+    PytestPluginSetup,
+)
+from pants.backend.python.target_types import Executable
+from pants.backend.python.util_rules.pex import (
+    PexRequest,
+    VenvPex,
+    VenvPexProcess,
+    rules as pex_rules,
+)
+from pants.engine.fs import CreateDigest, Digest, FileContent
+from pants.engine.rules import collect_rules, Get, rule
+from pants.engine.process import FallibleProcessResult, ProcessCacheScope
+from pants.engine.target import Target
+from pants.engine.unions import UnionRule
+from pants.util.logging import LogLevel
+
+from uses_services.exceptions import ServiceMissingError
+from uses_services.platform_rules import Platform
+from uses_services.scripts.is_st2cluster_running import (
+    __file__ as is_st2cluster_running_full_path,
+)
+from uses_services.target_types import UsesServicesField
+
+
+@dataclass(frozen=True)
+class UsesSt2ClusterRequest:
+    """One or more targets need a running st2 cluster with all st2* services."""
+
+    auth_host: str = "127.0.0.1"
+    auth_port: int = 9100
+    api_host: str = "127.0.0.1"
+    api_port: int = 9101
+    stream_host: str = "127.0.0.1"
+    stream_port: int = 9102
+
+    @property
+    def endpoints(self) -> tuple[tuple[str, str], ...]:
+        return (
+            (self.auth_host, str(self.auth_port)),
+            (self.api_host, str(self.api_port)),
+            (self.stream_host, str(self.stream_port)),
+        )
+
+    # @classmethod
+    # def from_env(cls, env: EnvironmentVars) -> UsesSt2ClusterRequest:
+    #    return cls()
+    # TODO: consider adding a from_env method using one or both of client => server vars:
+    #   ST2_CONFIG_FILE => ST2_CONFIG_PATH (used by many tests, so not safe) or
+    #                      ST2_CONF (only in launchdev.sh and st2ctl)
+    #   ST2_BASE_URL    => ST2_WEBUI__WEBUI_BASE_URL
+    #   ST2_API_URL     => ST2_AUTH__API_URL or
+    #                      http{'s' if ST2_API__USE_SSL else ''}://{ST2_API__HOST}:{ST2_API__PORT}
+    #   ST2_AUTH_URL    => http://{ST2_AUTH__HOST}:{ST2_AUTH__PORT}
+    #   ST2_STREAM_URL  => http://{ST2_STREAM__HOST}:{ST2_STREAM__PORT}
+    #   ST2_CACERT (might be needed if using a self-signed cert) => n/a
+    # These st2client env vars are irrelevant for the connectivity check:
+    #   ST2_AUTH_TOKEN or ST2_API_KEY
+    #   ST2_API_VERSION (always "v1" since we don't have anything else)
+
+
+@dataclass(frozen=True)
+class St2ClusterIsRunning:
+    pass
+
+
+class PytestUsesSt2ClusterRequest(PytestPluginSetupRequest):
+    @classmethod
+    def is_applicable(cls, target: Target) -> bool:
+        if not target.has_field(UsesServicesField):
+            return False
+        uses = target.get(UsesServicesField).value
+        return uses is not None and "st2cluster" in uses
+
+
+@rule(
+    desc="Ensure ST2 Cluster is running and accessible before running tests.",
+    level=LogLevel.DEBUG,
+)
+async def st2cluster_is_running_for_pytest(
+    request: PytestUsesSt2ClusterRequest,
+) -> PytestPluginSetup:
+    # this will raise an error if st2cluster is not running
+    _ = await Get(St2ClusterIsRunning, UsesSt2ClusterRequest())
+
+    return PytestPluginSetup()
+
+
+@rule(
+    desc="Test to see if ST2 Cluster is running and accessible.",
+    level=LogLevel.DEBUG,
+)
+async def st2cluster_is_running(
+    request: UsesSt2ClusterRequest, platform: Platform
+) -> St2ClusterIsRunning:
+    script_path = "./is_st2cluster_running.py"
+
+    # pants is already watching this directory as it is under a source root.
+    # So, we don't need to double watch with PathGlobs, just open it.
+    with open(is_st2cluster_running_full_path, "rb") as script_file:
+        script_contents = script_file.read()
+
+    script_digest = await Get(
+        Digest, CreateDigest([FileContent(script_path, script_contents)])
+    )
+    script_pex = await Get(
+        VenvPex,
+        PexRequest(
+            output_filename="script.pex",
+            internal_only=True,
+            sources=script_digest,
+            main=Executable(script_path),
+        ),
+    )
+
+    result = await Get(
+        FallibleProcessResult,
+        VenvPexProcess(
+            script_pex,
+            argv=[
+                host_or_port
+                for endpoint in request.endpoints
+                for host_or_port in endpoint
+            ],
+            input_digest=script_digest,
+            description="Checking to see if ST2 Cluster is up and accessible.",
+            # this can change from run to run, so don't cache results.
+            cache_scope=ProcessCacheScope.PER_SESSION,
+            level=LogLevel.DEBUG,
+        ),
+    )
+    is_running = result.exit_code == 0
+
+    if is_running:
+        return St2ClusterIsRunning()
+
+    # st2cluster is not running, so raise an error with instructions.
+    instructions = dedent(
+        """\
+        A full StackStorm cluster is required to run some integration tests.
+        To start the dev StackStorm cluster, run this from the repo root
+        (probably in a new terminal/window, as the output is quite verbose):
+
+        tools/launchdev.sh start -x
+
+        This runs each StackStorm microservice in a tmux session. You can
+        inspect the logs for this service in the `logs/` directory.
+
+        If tmux is not installed, please install it with a package manager,
+        or use vagrant for local development with something like:
+
+        vagrant init stackstorm/st2
+        vagrant up
+        vagrant ssh
+
+        Please see: https://docs.stackstorm.com/install/vagrant.html
+        """
+    )
+    raise ServiceMissingError(
+        service="st2cluster",
+        platform=platform,
+        instructions=instructions,
+        msg=f"The dev StackStorm cluster seems to be down.\n{instructions}",
+    )
+
+
+def rules():
+    return [
+        *collect_rules(),
+        UnionRule(PytestPluginSetupRequest, PytestUsesSt2ClusterRequest),
+        *pex_rules(),
+    ]