Skip to content

Commit

Permalink
Refactor codebase into core and server subpackages
Browse files Browse the repository at this point in the history
- Moved shared logic (config, exceptions, ignore_patterns, etc.) into `core/`.
- Created `server/` to host FastAPI modules, static files, and templates.
- Updated import paths for CLI and tests to reference `core.*` and `server.*`.
- Changed static references from `"/static"` to `"/server/static"` and templates from `"/templates"` to `"/server/templates"`.
- Adjusted references in `base.jinja`, `server/main.py`, `server/query_processor.py`, and routers to reflect the new server paths.
- Ensured Docker builds now correctly serve static and template files from `server/`.
  • Loading branch information
filipchristiansen committed Jan 17, 2025
1 parent 3ce8e7e commit 62e7d53
Show file tree
Hide file tree
Showing 43 changed files with 84 additions and 93 deletions.
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -41,4 +41,4 @@ USER appuser

EXPOSE 8000

CMD ["python", "-m", "uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["python", "-m", "uvicorn", "server.main:app", "--host", "0.0.0.0", "--port", "8000"]
Empty file added src/core/__init__.py
Empty file.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from pathlib import Path
from typing import Any

from gitingest.exceptions import InvalidNotebookError
from core.exceptions import InvalidNotebookError


def process_notebook(file: Path, include_output: bool = True) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,8 @@

import tiktoken

from gitingest.exceptions import (
AlreadyVisitedError,
InvalidNotebookError,
MaxFileSizeReachedError,
MaxFilesReachedError,
)
from gitingest.notebook_utils import process_notebook
from core.exceptions import AlreadyVisitedError, InvalidNotebookError, MaxFileSizeReachedError, MaxFilesReachedError
from core.notebook_utils import process_notebook

MAX_FILE_SIZE = 10 * 1024 * 1024 # 10 MB
MAX_DIRECTORY_DEPTH = 20 # Maximum depth of directory traversal
Expand Down
8 changes: 4 additions & 4 deletions src/gitingest/query_parser.py → src/core/query_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@
from typing import Any
from urllib.parse import unquote, urlparse

from config import TMP_BASE_PATH
from gitingest.exceptions import InvalidPatternError
from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
from gitingest.repository_clone import _check_repo_exists, fetch_remote_branch_list
from core.config import TMP_BASE_PATH
from core.exceptions import InvalidPatternError
from core.ignore_patterns import DEFAULT_IGNORE_PATTERNS
from core.repository_clone import _check_repo_exists, fetch_remote_branch_list

HEX_DIGITS: set[str] = set(string.hexdigits)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import asyncio
from dataclasses import dataclass

from gitingest.utils import async_timeout
from core.utils import async_timeout

TIMEOUT: int = 20

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import inspect
import shutil

from config import TMP_BASE_PATH
from gitingest.query_ingestion import run_ingest_query
from gitingest.query_parser import parse_query
from gitingest.repository_clone import CloneConfig, clone_repo
from core.config import TMP_BASE_PATH
from core.query_ingestion import run_ingest_query
from core.query_parser import parse_query
from core.repository_clone import CloneConfig, clone_repo


async def ingest(
Expand Down
2 changes: 1 addition & 1 deletion src/gitingest/utils.py → src/core/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from collections.abc import Awaitable, Callable
from typing import ParamSpec, TypeVar

from gitingest.exceptions import AsyncTimeoutError
from core.exceptions import AsyncTimeoutError

T = TypeVar("T")
P = ParamSpec("P")
Expand Down
8 changes: 4 additions & 4 deletions src/gitingest/__init__.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
""" Gitingest: A package for ingesting data from Git repositories. """

from gitingest.query_ingestion import run_ingest_query
from gitingest.query_parser import parse_query
from gitingest.repository_clone import clone_repo
from gitingest.repository_ingest import ingest
from core.query_ingestion import run_ingest_query
from core.query_parser import parse_query
from core.repository_clone import clone_repo
from core.repository_ingest import ingest

__all__ = ["run_ingest_query", "clone_repo", "parse_query", "ingest"]
4 changes: 2 additions & 2 deletions src/gitingest/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@

import click

from gitingest.query_ingestion import MAX_FILE_SIZE
from gitingest.repository_ingest import ingest
from core.query_ingestion import MAX_FILE_SIZE
from core.repository_ingest import ingest


@click.command()
Expand Down
7 changes: 0 additions & 7 deletions src/routers/__init__.py

This file was deleted.

Empty file added src/server/__init__.py
Empty file.
12 changes: 6 additions & 6 deletions src/main.py → src/server/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
from slowapi.errors import RateLimitExceeded
from starlette.middleware.trustedhost import TrustedHostMiddleware

from config import DELETE_REPO_AFTER, TMP_BASE_PATH
from routers import download, dynamic, index
from server_utils import limiter
from core.config import DELETE_REPO_AFTER, TMP_BASE_PATH
from server.routers import download, dynamic, index
from server.server_utils import limiter

# Load environment variables from .env file
load_dotenv()
Expand Down Expand Up @@ -156,7 +156,7 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp
app.add_exception_handler(RateLimitExceeded, rate_limit_exception_handler)

# Mount static files to serve CSS, JS, and other static assets
app.mount("/static", StaticFiles(directory="static"), name="static")
app.mount("/server/static", StaticFiles(directory="server/static"), name="static")

# Set up API analytics middleware if an API key is provided
if app_analytics_key := os.getenv("API_ANALYTICS_KEY"):
Expand All @@ -175,7 +175,7 @@ async def rate_limit_exception_handler(request: Request, exc: Exception) -> Resp
app.add_middleware(TrustedHostMiddleware, allowed_hosts=allowed_hosts)

# Set up template rendering
templates = Jinja2Templates(directory="templates")
templates = Jinja2Templates(directory="server/templates")


@app.get("/health")
Expand Down Expand Up @@ -235,7 +235,7 @@ async def robots() -> FileResponse:
FileResponse
The `robots.txt` file located in the static directory.
"""
return FileResponse("static/robots.txt")
return FileResponse("server/static/robots.txt")


# Include routers for modular endpoints
Expand Down
12 changes: 6 additions & 6 deletions src/query_processor.py → src/server/query_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@
from fastapi.templating import Jinja2Templates
from starlette.templating import _TemplateResponse

from config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE
from gitingest.query_ingestion import run_ingest_query
from gitingest.query_parser import parse_query
from gitingest.repository_clone import CloneConfig, clone_repo
from server_utils import Colors, log_slider_to_size
from core.config import EXAMPLE_REPOS, MAX_DISPLAY_SIZE
from core.query_ingestion import run_ingest_query
from core.query_parser import parse_query
from core.repository_clone import CloneConfig, clone_repo
from server.server_utils import Colors, log_slider_to_size

templates = Jinja2Templates(directory="templates")
templates = Jinja2Templates(directory="server/templates")


async def process_query(
Expand Down
7 changes: 7 additions & 0 deletions src/server/routers/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
""" This module contains the routers for the FastAPI application. """

from server.routers.download import router as download
from server.routers.dynamic import router as dynamic
from server.routers.index import router as index

__all__ = ["download", "dynamic", "index"]
2 changes: 1 addition & 1 deletion src/routers/download.py → src/server/routers/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from fastapi import APIRouter, HTTPException
from fastapi.responses import Response

from config import TMP_BASE_PATH
from core.config import TMP_BASE_PATH

router = APIRouter()

Expand Down
6 changes: 3 additions & 3 deletions src/routers/dynamic.py → src/server/routers/dynamic.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates

from query_processor import process_query
from server_utils import limiter
from server.query_processor import process_query
from server.server_utils import limiter

router = APIRouter()
templates = Jinja2Templates(directory="templates")
templates = Jinja2Templates(directory="server/templates")


@router.get("/{full_path:path}")
Expand Down
8 changes: 4 additions & 4 deletions src/routers/index.py → src/server/routers/index.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
from fastapi.responses import HTMLResponse
from fastapi.templating import Jinja2Templates

from config import EXAMPLE_REPOS
from query_processor import process_query
from server_utils import limiter
from core.config import EXAMPLE_REPOS
from server.query_processor import process_query
from server.server_utils import limiter

router = APIRouter()
templates = Jinja2Templates(directory="templates")
templates = Jinja2Templates(directory="server/templates")


@router.get("/", response_class=HTMLResponse)
Expand Down
File renamed without changes.
File renamed without changes
File renamed without changes
File renamed without changes.
File renamed without changes
File renamed without changes.
File renamed without changes
File renamed without changes.
File renamed without changes.
12 changes: 6 additions & 6 deletions src/templates/base.jinja → src/server/templates/base.jinja
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,22 @@
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="icon" type="image/x-icon" href="/static/favicon.ico">
<link rel="icon" type="image/x-icon" href="/server/static/favicon.ico">
<!-- Search Engine Meta Tags -->
<meta name="description"
content="Replace 'hub' with 'ingest' in any GitHub URL for a prompt-friendly text.">
<meta name="keywords"
content="Gitingest, AI tools, LLM integration, Ingest, Digest, Context, Prompt, Git workflow, codebase extraction, Git repository, Git automation, Summarize, prompt-friendly">
<meta name="robots" content="index, follow">
<!-- Favicons -->
<link rel="icon" type="image/svg+xml" href="/static/favicon.svg">
<link rel="icon" type="image/svg+xml" href="/server/static/favicon.svg">
<link rel="icon"
type="image/png"
sizes="64x64"
href="/static/favicon-64.png">
href="/server/static/favicon-64.png">
<link rel="apple-touch-icon"
sizes="180x180"
href="/static/apple-touch-icon.png">
href="/server/static/apple-touch-icon.png">
<!-- Web App Meta -->
<meta name="apple-mobile-web-app-title" content="Gitingest">
<meta name="application-name" content="Gitingest">
Expand All @@ -31,12 +31,12 @@
content="Replace 'hub' with 'ingest' in any GitHub URL for a prompt-friendly text.">
<meta property="og:type" content="website">
<meta property="og:url" content="{{ request.url }}">
<meta property="og:image" content="/static/og-image.png">
<meta property="og:image" content="/server/static/og-image.png">
<title>
{% block title %}Gitingest{% endblock %}
</title>
<script src="https://cdn.tailwindcss.com"></script>
<script src="/static/js/utils.js"></script>
<script src="/server/static/js/utils.js"></script>
{% block extra_head %}{% endblock %}
</head>
<body class="bg-[#FFFDF8] min-h-screen flex flex-col">
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Empty file added tests/core/__init__.py
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest

from gitingest.query_parser import parse_query
from core.query_parser import parse_query


@pytest.mark.parametrize(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@

import pytest

from gitingest.ignore_patterns import DEFAULT_IGNORE_PATTERNS
from gitingest.query_parser import _parse_patterns, _parse_repo_source, parse_query
from core.ignore_patterns import DEFAULT_IGNORE_PATTERNS
from core.query_parser import _parse_patterns, _parse_repo_source, parse_query


async def test_parse_url_valid_https() -> None:
Expand Down Expand Up @@ -110,11 +110,9 @@ async def test_parse_url_with_subpaths() -> None:
Verifies that user name, repository name, branch, and subpath are correctly extracted.
"""
url = "https://github.com/user/repo/tree/main/subdir/file"
with patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
with patch("core.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
mock_run_git_command.return_value = (b"refs/heads/main\nrefs/heads/dev\nrefs/heads/feature-branch\n", b"")
with patch(
"gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock
) as mock_fetch_branches:
with patch("core.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches:
mock_fetch_branches.return_value = ["main", "dev", "feature-branch"]
result = await _parse_repo_source(url)
assert result["user_name"] == "user"
Expand Down Expand Up @@ -235,11 +233,9 @@ async def test_parse_url_branch_and_commit_distinction() -> None:
url_branch = "https://github.com/user/repo/tree/main"
url_commit = "https://github.com/user/repo/tree/abcd1234abcd1234abcd1234abcd1234abcd1234"

with patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
with patch("core.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command:
mock_run_git_command.return_value = (b"refs/heads/main\nrefs/heads/dev\nrefs/heads/feature-branch\n", b"")
with patch(
"gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock
) as mock_fetch_branches:
with patch("core.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches:
mock_fetch_branches.return_value = ["main", "dev", "feature-branch"]

result_branch = await _parse_repo_source(url_branch)
Expand Down Expand Up @@ -309,7 +305,7 @@ async def test_parse_repo_source_with_failed_git_command(url, expected_branch, e
Test `_parse_repo_source` when git command fails.
Verifies that the function returns the first path component as the branch.
"""
with patch("gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches:
with patch("core.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches:
mock_fetch_branches.side_effect = Exception("Failed to fetch branch list")

result = await _parse_repo_source(url)
Expand All @@ -332,8 +328,8 @@ async def test_parse_repo_source_with_failed_git_command(url, expected_branch, e
)
async def test_parse_repo_source_with_various_url_patterns(url, expected_branch, expected_subpath):
with (
patch("gitingest.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command,
patch("gitingest.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches,
patch("core.repository_clone._run_git_command", new_callable=AsyncMock) as mock_run_git_command,
patch("core.repository_clone.fetch_remote_branch_list", new_callable=AsyncMock) as mock_fetch_branches,
):

mock_run_git_command.return_value = (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import pytest

from gitingest.notebook_utils import process_notebook
from core.notebook_utils import process_notebook


def test_process_notebook_all_cells(write_notebook):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from typing import Any
from unittest.mock import patch

from gitingest.query_ingestion import _extract_files_content, _read_file_content, _scan_directory, run_ingest_query
from core.query_ingestion import _extract_files_content, _read_file_content, _scan_directory, run_ingest_query


def test_scan_directory(temp_directory: Path, sample_query: dict[str, Any]) -> None:
Expand Down Expand Up @@ -43,7 +43,7 @@ def test_read_file_content_with_notebook(tmp_path: Path):
notebook_path.write_text("{}", encoding="utf-8") # minimal JSON

# Patch the symbol as it is used in query_ingestion
with patch("gitingest.query_ingestion.process_notebook") as mock_process:
with patch("core.query_ingestion.process_notebook") as mock_process:
_read_file_content(notebook_path)
mock_process.assert_called_once_with(notebook_path)

Expand All @@ -52,7 +52,7 @@ def test_read_file_content_with_non_notebook(tmp_path: Path):
py_file_path = tmp_path / "dummy_file.py"
py_file_path.write_text("print('Hello')", encoding="utf-8")

with patch("gitingest.query_ingestion.process_notebook") as mock_process:
with patch("core.query_ingestion.process_notebook") as mock_process:
_read_file_content(py_file_path)
mock_process.assert_not_called()

Expand Down
Loading

0 comments on commit 62e7d53

Please sign in to comment.