From 65c8f6b235a3576b1a8c4cf6c6f804373b98b767 Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Sun, 22 Dec 2024 21:16:47 -0500 Subject: [PATCH 1/2] Initial setup of cache --- backend/app/api/deps.py | 15 ++++++++ backend/app/api/routes/health.py | 12 +++++-- backend/app/core/cache.py | 35 +++++++++++++++++++ backend/app/core/config.py | 4 +++ backend/app/main.py | 8 +++++ backend/pyproject.toml | 1 + backend/tests/api/routes/test_health_route.py | 1 + backend/tests/conftest.py | 12 +++++++ backend/uv.lock | 11 ++++++ docker-compose.ci.yml | 11 ++++++ docker-compose.yml | 9 +++++ justfile | 6 ++-- 12 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 backend/app/core/cache.py diff --git a/backend/app/api/deps.py b/backend/app/api/deps.py index 8d74d61..0440e53 100644 --- a/backend/app/api/deps.py +++ b/backend/app/api/deps.py @@ -3,6 +3,7 @@ import asyncpg import jwt +import valkey.asyncio as valkey from fastapi import Depends, HTTPException from fastapi.security import OAuth2PasswordBearer from jwt.exceptions import InvalidTokenError @@ -15,6 +16,7 @@ HTTP_503_SERVICE_UNAVAILABLE, ) +from app.core.cache import cache from app.core.config import settings from app.core.db import db from app.core.security import ALGORITHM @@ -39,6 +41,19 @@ async def get_db_conn() -> AsyncGenerator[asyncpg.Connection, None]: DbConn = Annotated[asyncpg.Connection, Depends(get_db_conn)] +async def get_cache_client() -> AsyncGenerator[valkey.Valkey, None]: + if cache.client is None: + logger.error("No cache client created") + raise HTTPException( + status_code=HTTP_503_SERVICE_UNAVAILABLE, detail="The cache is currently unavailable" + ) + + yield cache.client + + +CacheClient = Annotated[valkey.Valkey, Depends(get_cache_client)] + + async def get_current_user(conn: DbConn, token: TokenDep) -> UserInDb: try: logger.debug("Decoding JWT token") diff --git a/backend/app/api/routes/health.py b/backend/app/api/routes/health.py index e24bfcc..3c79420 100644 --- a/backend/app/api/routes/health.py +++ b/backend/app/api/routes/health.py @@ -1,6 +1,6 @@ from loguru import logger -from app.api.deps import DbConn +from app.api.deps import CacheClient, DbConn from app.core.config import settings from app.core.utils import APIRouter from app.services.db_services import ping @@ -11,7 +11,7 @@ @router.get("/") -async def health(*, db_conn: DbConn) -> dict[str, str]: +async def health(*, cache_client: CacheClient, db_conn: DbConn) -> dict[str, str]: """Check the health of the server.""" logger.debug("Checking health") @@ -25,4 +25,12 @@ async def health(*, db_conn: DbConn) -> dict[str, str]: logger.error(f"Unable to ping the database: {e}") health["db"] = "unhealthy" + logger.debug("Checking cache health") + try: + await cache_client.ping() + health["cache"] = "healthy" + except Exception as e: + logger.error(f"Unable to ping the cache server: {e}") + health["cache"] = "unhealthy" + return health diff --git a/backend/app/core/cache.py b/backend/app/core/cache.py new file mode 100644 index 0000000..99183f3 --- /dev/null +++ b/backend/app/core/cache.py @@ -0,0 +1,35 @@ +import valkey.asyncio as valkey + +from app.core.config import settings + + +class Cache: + def __init__(self) -> None: + self._pool: valkey.ConnectionPool | None = None + self.client: valkey.Valkey | None = None + + async def create_client(self) -> None: + self._pool = await self._create_pool() + self.client = valkey.Valkey.from_pool(self._pool) + + async def close_client(self) -> None: + if self.client: + await self.client.aclose() + + if self._pool: + await self._pool.aclose() + + async def _create_pool(self) -> valkey.ConnectionPool: + return valkey.ConnectionPool( + host=settings.VALKEY_HOST, + port=settings.VALKEY_PORT, + password=settings.VALKEY_PASSWORD.get_secret_value(), + db=0, + ) + + async def _close_pool(self) -> None: + if self._pool: + await self._pool.aclose() + + +cache = Cache() diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 2b0ca93..91f0b5f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -45,6 +45,9 @@ class Settings(BaseSettings): POSTGRES_USER: str POSTGRES_PASSWORD: SecretStr POSTGRES_DB: str = "scan" + VALKEY_HOST: str + VALKEY_PASSWORD: SecretStr + VALKEY_PORT: int = 6379 OPENAI_API_KEY: SecretStr DLPFC_MODEL: SecretStr = SecretStr("gpt-3.5-turbo") VMPFC_MODEL: SecretStr = SecretStr("gpt-3.5-turbo") @@ -89,6 +92,7 @@ def _enforce_non_default_secrets(self) -> Self: ) self._check_default_secret("POSTGRES_USER", self.POSTGRES_USER) self._check_default_secret("POSTGRES_PASSWORD", self.POSTGRES_PASSWORD.get_secret_value()) + self._check_default_secret("VALKEY_PASSWORD", self.VALKEY_PASSWORD.get_secret_value()) return self diff --git a/backend/app/main.py b/backend/app/main.py index cf54b6e..3ea3873 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -7,6 +7,7 @@ from loguru import logger from app.api.router import api_router +from app.core.cache import cache from app.core.config import settings from app.core.db import db @@ -40,6 +41,13 @@ async def lifespan(_: FastAPI) -> AsyncGenerator: logger.error(f"Error creating first superuser: {e}") raise e + logger.info("Initializing cache client") + try: + await cache.create_client() + except Exception as e: + logger.error(f"Error creating cache client: {e}") + raise + yield logger.info("Closing database connection pool") try: diff --git a/backend/pyproject.toml b/backend/pyproject.toml index be89748..5abf9f7 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "python-multipart==0.0.20", "uvicorn==0.34.0", "uvloop==0.21.0", + "valkey==6.0.2", ] [dependency-groups] diff --git a/backend/tests/api/routes/test_health_route.py b/backend/tests/api/routes/test_health_route.py index ad076e8..2d29746 100644 --- a/backend/tests/api/routes/test_health_route.py +++ b/backend/tests/api/routes/test_health_route.py @@ -4,3 +4,4 @@ async def test_health(test_client): assert result.status_code == 200 assert result.json()["server"] == "healthy" assert result.json()["db"] == "healthy" + assert result.json()["cache"] == "healthy" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 59d4f73..b1a0200 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,6 +1,7 @@ import pytest from httpx import ASGITransport, AsyncClient +from app.core.cache import cache from app.core.config import settings from app.core.db import db from app.main import app @@ -33,6 +34,17 @@ async def test_db(): await db.close_pool() +@pytest.fixture(autouse=True) +async def test_cache(): + await cache.create_client() + yield cache + if cache.client: + await cache.create_client() + + await cache.client.flushall() # type: ignore + await cache.close_client() + + @pytest.fixture async def test_client(): async with AsyncClient( diff --git a/backend/uv.lock b/backend/uv.lock index a964a5f..a52f744 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -98,6 +98,7 @@ dependencies = [ { name = "python-multipart" }, { name = "uvicorn" }, { name = "uvloop" }, + { name = "valkey" }, ] [package.dev-dependencies] @@ -129,6 +130,7 @@ requires-dist = [ { name = "python-multipart", specifier = "==0.0.20" }, { name = "uvicorn", specifier = "==0.34.0" }, { name = "uvloop", specifier = "==0.21.0" }, + { name = "valkey", specifier = "==6.0.2" }, ] [package.metadata.requires-dev] @@ -1250,6 +1252,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/63/9a/0962b05b308494e3202d3f794a6e85abe471fe3cafdbcf95c2e8c713aabd/uvloop-0.21.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a5c39f217ab3c663dc699c04cbd50c13813e31d917642d459fdcec07555cc553", size = 4660018 }, ] +[[package]] +name = "valkey" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/f7/b552b7a67017e6233cd8a3b783ce8c4b548e29df98daedd7fb4c4c2cc8f8/valkey-6.0.2.tar.gz", hash = "sha256:dc2e91512b82d1da0b91ab0cdbd8c97c0c0250281728cb32f9398760df9caeae", size = 4602149 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/cb/b1eac0fe9cbdbba0a5cf189f5778fe54ba7d7c9f26c2f62ca8d759b38f52/valkey-6.0.2-py3-none-any.whl", hash = "sha256:dbbdd65439ee0dc5689502c54f1899504cc7268e85cb7fe8935f062178ff5805", size = 260101 }, +] + [[package]] name = "virtualenv" version = "20.27.1" diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 5127e21..8810b10 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -6,6 +6,7 @@ services: container_name: backend depends_on: - db + - valkey ports: - "8000:8000" environment: @@ -16,6 +17,8 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=test_password - POSTGRES_DB=scan + - VALKEY_HOST=127.0.0.1 + - VALKEY_PASSWORD=test_password - OPENAI_API_KEY=someKey db: image: postgres:17-alpine @@ -30,6 +33,14 @@ services: volumes: - db-data:/var/lib/postgresql/data + valkey: + image: valkey/valkey:8-alpine + expose: + - 6379 + ports: + - 6379:6379 + command: valkey-server --requirepass test_password + volumes: db-data: diff --git a/docker-compose.yml b/docker-compose.yml index 4e7f170..e4d0aa3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,7 @@ services: container_name: backend depends_on: - db + - valkey ports: - "8000:8000" env_file: @@ -25,6 +26,14 @@ services: volumes: - db-data:/var/lib/postgresql/data + valkey: + image: valkey/valkey:8-alpine + expose: + - 6379 + ports: + - 6379:6379 + command: valkey-server --requirepass test_password + volumes: db-data: diff --git a/justfile b/justfile index 18d316b..583a010 100644 --- a/justfile +++ b/justfile @@ -61,15 +61,15 @@ cd frontend && \ npm run start -@fronend-line: +@frontend-lint: cd frontend && \ npm run lint @docker-up: - docker compose up + docker compose up --build @docker-up-backend-dev: - docker compose up db + docker compose up db valkey @docker-up-detached: docker compose up -d From 2f0cab0840915177cde189ddf0d3fd75b7d54b7e Mon Sep 17 00:00:00 2001 From: Paul Sanders Date: Sun, 22 Dec 2024 21:21:07 -0500 Subject: [PATCH 2/2] Fix mypy errors --- backend/tests/core/test_config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/tests/core/test_config.py b/backend/tests/core/test_config.py index bc53ff0..1e35f24 100644 --- a/backend/tests/core/test_config.py +++ b/backend/tests/core/test_config.py @@ -13,6 +13,8 @@ def test_temperature(temperature): POSTGRES_HOST="some_host", POSTGRES_USER="pg", POSTGRES_PASSWORD=SecretStr("pgpassword"), + VALKEY_HOST="valkey", + VALKEY_PASSWORD=SecretStr("valkeypassword"), OPENAI_API_KEY=SecretStr("some_key"), TEMPERATURE=temperature, ) @@ -30,6 +32,8 @@ def test_invalid_temperature(temperature): POSTGRES_HOST="some_host", POSTGRES_USER="pg", POSTGRES_PASSWORD=SecretStr("pgpassword"), + VALKEY_HOST="valkey", + VALKEY_PASSWORD=SecretStr("valkeypassword"), OPENAI_API_KEY=SecretStr("some_key"), TEMPERATURE=temperature, )