Merge pull request #2 from lsst-dm/tickets/DM-47606

DM-47606: Initial code for s3proxy.

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.7.4
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

requirements/main.in

@@ -12,6 +12,8 @@ fastapi>=0.100
 uvicorn[standard]
 
 # Other dependencies.
+boto3
+lsst-resources
 pydantic>2
 pydantic-settings
 safir>=5

requirements/tox.txt

@@ -0,0 +1,86 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --universal --generate-hashes --output-file requirements/tox.txt requirements/tox.in
+cachetools==5.5.0 \
+    --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \
+    --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a
+    # via tox
+chardet==5.2.0 \
+    --hash=sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7 \
+    --hash=sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970
+    # via tox
+colorama==0.4.6 \
+    --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
+    --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+    # via
+    #   -c requirements/dev.txt
+    #   -c requirements/main.txt
+    #   tox
+distlib==0.3.9 \
+    --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \
+    --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403
+    # via virtualenv
+filelock==3.16.1 \
+    --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
+    --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
+    # via
+    #   tox
+    #   virtualenv
+packaging==24.2 \
+    --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \
+    --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f
+    # via
+    #   -c requirements/dev.txt
+    #   -c requirements/main.txt
+    #   pyproject-api
+    #   tox
+    #   tox-uv
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via
+    #   tox
+    #   virtualenv
+pluggy==1.5.0 \
+    --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
+    --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
+    # via
+    #   -c requirements/dev.txt
+    #   tox
+pyproject-api==1.8.0 \
+    --hash=sha256:3d7d347a047afe796fd5d1885b1e391ba29be7169bd2f102fcd378f04273d228 \
+    --hash=sha256:77b8049f2feb5d33eefcc21b57f1e279636277a8ac8ad6b5871037b243778496
+    # via tox
+tox==4.23.2 \
+    --hash=sha256:452bc32bb031f2282881a2118923176445bac783ab97c874b8770ab4c3b76c38 \
+    --hash=sha256:86075e00e555df6e82e74cfc333917f91ecb47ffbc868dcafbd2672e332f4a2c
+    # via
+    #   -r requirements/tox.in
+    #   tox-uv
+tox-uv==1.16.0 \
+    --hash=sha256:71b2e2fa6c35c1360b91a302df1d65b3e5a1f656b321c5ebf7b84545804c9f01 \
+    --hash=sha256:e6f0b525a687e745ab878d07cbf5c7e85d582028d4a7c8935f95e84350651432
+    # via -r requirements/tox.in
+uv==0.5.2 \
+    --hash=sha256:15c7ffa08ae21abd221dbdf9ba25c8969235f587cec6df8035552434e5ca1cc5 \
+    --hash=sha256:2597e91be45b3f4458d0d16a5a1cda7e93af7d6dbfddf251aae5377f9187fa88 \
+    --hash=sha256:27d666da8fbb0f87d9df67abf9feea0da4ee1336730f2c4be29a11f3feaa0a29 \
+    --hash=sha256:374e9498e155fcaa8728a6770b84f03781106d705332f4ec059e1cc93c8f4d8a \
+    --hash=sha256:5052758d374dd769efd0c70b4789ffb08439567eb114ad8fe728536bb5cc5299 \
+    --hash=sha256:675ca34829ceca3e9de395cf05e8f881334a24488f97dd923c463830270d52a7 \
+    --hash=sha256:67776d34cba359c63919c5ad50331171261d2ec7a83fd07f032eb8cc22e22b8e \
+    --hash=sha256:71467545d51883d1af7094c8f6da69b55e7d49b742c2dc707d644676dcb66515 \
+    --hash=sha256:772b32d157ec8f27c0099ecac94cf5cd298bce72f1a1f512205591de4e9f0c5c \
+    --hash=sha256:7bde66f13571e437fd45f32f5742ab53d5e011b4edb1c74cb74cb8b1cbb828b5 \
+    --hash=sha256:89e60ad9601f35f187326de84f35e7517c6eb1438359da42ec85cfd9c1895957 \
+    --hash=sha256:a4d4fdad03e6dc3e8216192b8a12bcf2c71c8b12046e755575c7f262cbb61924 \
+    --hash=sha256:a8a9897dd7657258c53f41aecdbe787da99f4fc0775f19826ab65cc0a7136cbf \
+    --hash=sha256:c9795b990fb0b2a18d3a8cef8822e13c6a6f438bc16d34ccf01d931c76cfd5da \
+    --hash=sha256:cfba5b0070652da4174083b78852f3ab3d262ba1c8b63a4d5ae497263b02b834 \
+    --hash=sha256:d0834c6b37750c045bbea80600d3ae3e95becc4db148f5c0d0bc3ec6a7924e8f \
+    --hash=sha256:d1fe4e025dbb9ec5c9250bfc1231847b8487706538f94d10c769f0a54db3e0af \
+    --hash=sha256:dfcd8275ff8cb59d5f26f826a44270b2fe8f38aa7188d7355c48d3e9b759d0c0
+    # via tox-uv
+virtualenv==20.27.1 \
+    --hash=sha256:142c6be10212543b32c6c45d3d3893dff89112cc588b7d0879ae5a1ec03a47ba \
+    --hash=sha256:f11f1b8a29525562925f745563bfd48b189450f61fb34c4f9cc79dd5aa32a1f4
+    # via tox

src/s3proxy/config.py

@@ -14,9 +14,7 @@ class Config(BaseSettings):
 
     name: str = Field("s3proxy", title="Name of application")
 
-    path_prefix: str = Field(
-        "/s3proxy", title="URL prefix for application"
-    )
+    path_prefix: str = Field("/s3proxy", title="URL prefix for application")
 
     profile: Profile = Field(
         Profile.development, title="Application logging profile"

src/s3proxy/handlers/external.py

@@ -1,16 +1,19 @@
 """Handlers for the app's external root, ``/s3proxy/``."""
 
+import mimetypes
 from typing import Annotated
 
-from fastapi import APIRouter, Depends
-from safir.dependencies.logger import logger_dependency
+from fastapi import APIRouter, Depends, Response
+from fastapi.responses import JSONResponse
+from lsst.resources import ResourcePath
+from safir.dependencies.gafaelfawr import auth_logger_dependency
 from safir.metadata import get_metadata
 from structlog.stdlib import BoundLogger
 
 from ..config import config
 from ..models import Index
 
-__all__ = ["get_index", "external_router"]
+__all__ = ["get_index", "get_s3", "external_router"]
 
 external_router = APIRouter()
 """FastAPI router for all external handlers."""
@@ -27,7 +30,7 @@
     summary="Application metadata",
 )
 async def get_index(
-    logger: Annotated[BoundLogger, Depends(logger_dependency)],
+    logger: Annotated[BoundLogger, Depends(auth_logger_dependency)],
 ) -> Index:
     """GET ``/s3proxy/`` (the app's external root).
 
@@ -50,3 +53,33 @@ async def get_index(
         application_name=config.name,
     )
     return Index(metadata=metadata)
+
+
+@external_router.get(
+    "/s3/{bucket}/{key:path}",
+    description=(
+        "Return an S3 object's contents.  ``bucket`` can contain ``profile@``."
+    ),
+    summary="Object contents",
+)
+async def get_s3(
+    bucket: str,
+    key: str,
+    logger: Annotated[BoundLogger, Depends(auth_logger_dependency)],
+) -> Response:
+    """GET ``/s3proxy/s3/{bucket}/{key:path}``.
+
+    This returns the contents of an s3 object
+    """
+    path = f"s3://{bucket}/{key}"
+    logger.info("s3 request")
+    rp = ResourcePath(path)
+    mimetype, encoding = mimetypes.guess_type(path)
+    if mimetype is None:
+        mimetype = "application/octet-stream"
+    try:
+        return Response(content=rp.read(), media_type=mimetype)
+    except FileNotFoundError:
+        return JSONResponse(
+            status_code=404, content={"message": f"Not found: {path}"}
+        )

tests/handlers/external_test.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import pytest
+from botocore.exceptions import NoCredentialsError
 from httpx import AsyncClient
 
 from s3proxy.config import config
@@ -11,7 +12,9 @@
 @pytest.mark.asyncio
 async def test_get_index(client: AsyncClient) -> None:
     """Test ``GET /s3proxy/``."""
-    response = await client.get("/s3proxy/")
+    response = await client.get(
+        "/s3proxy/", headers={"X-Auth-Request-User": "test"}
+    )
     assert response.status_code == 200
     data = response.json()
     metadata = data["metadata"]
@@ -20,3 +23,12 @@ async def test_get_index(client: AsyncClient) -> None:
     assert isinstance(metadata["description"], str)
     assert isinstance(metadata["repository_url"], str)
     assert isinstance(metadata["documentation_url"], str)
+
+
+@pytest.mark.asyncio
+async def test_get_s3(client: AsyncClient) -> None:
+    """Test ``GET /s3proxy/s3``."""
+    with pytest.raises(NoCredentialsError):
+        await client.get(
+            "/s3proxy/s3/bucket/key", headers={"X-Auth-Request-User": "test"}
+        )

tox.ini

@@ -34,3 +34,7 @@ commands = pre-commit run --all-files
 description = Run the development server with auto-reload for code changes.
 usedevelop = true
 commands = uvicorn s3proxy.main:app --reload
+pass_env =
+    LSST_RESOURCES_S3_PROFILE_*
+    S3_ENDPOINT_URL
+    AWS_SHARED_CREDENTIALS_FILE