From af1308314ac184ceb44f1138a524b248648eac27 Mon Sep 17 00:00:00 2001 From: Alexandre Negrel Date: Mon, 7 Oct 2024 12:42:05 +0200 Subject: [PATCH] add maintainer and developer documentation --- cmd/README.md | 3 ++ cmd/addevents/README.md | 7 +++ cmd/server/README.md | 10 ++++ config/README.md | 4 ++ docs/README.md | 12 +++++ docs/dev.md | 108 ++++++++++++++++++++++++++++++++++++++++ docs/maintainer.md | 44 ++++++++++++++++ docs/repository.md | 41 +++++++++++++++ pkg/README.md | 15 ++++++ pkg/clickhouse/ch.go | 3 ++ pkg/config/config.go | 2 + pkg/embedded/doc.go | 2 + pkg/event/doc.go | 2 + pkg/grafana/client.go | 2 + pkg/handlers/doc.go | 2 + pkg/log/logger.go | 1 + pkg/middlewares/doc.go | 2 + pkg/secret/secret.go | 1 + pkg/services/README.md | 8 +++ pkg/wired/app.go | 1 + pkg/wired/doc.go | 3 ++ tests/README.md | 3 ++ 22 files changed, 276 insertions(+) create mode 100644 cmd/README.md create mode 100644 cmd/addevents/README.md create mode 100644 cmd/server/README.md create mode 100644 config/README.md create mode 100644 docs/README.md create mode 100644 docs/dev.md create mode 100644 docs/maintainer.md create mode 100644 docs/repository.md create mode 100644 pkg/README.md create mode 100644 pkg/config/config.go create mode 100644 pkg/embedded/doc.go create mode 100644 pkg/event/doc.go create mode 100644 pkg/handlers/doc.go create mode 100644 pkg/middlewares/doc.go create mode 100644 pkg/services/README.md create mode 100644 pkg/wired/doc.go create mode 100644 tests/README.md diff --git a/cmd/README.md b/cmd/README.md new file mode 100644 index 0000000..9e94f01 --- /dev/null +++ b/cmd/README.md @@ -0,0 +1,3 @@ +# `cmd` - Go binaries + +This directory holds `main` packages of our Go binaries. diff --git a/cmd/addevents/README.md b/cmd/addevents/README.md new file mode 100644 index 0000000..7ac7c63 --- /dev/null +++ b/cmd/addevents/README.md @@ -0,0 +1,7 @@ +# `addevents` - Add fictives events to ClickHouse + +This directory contains a small CLI utils to add fictive data to ClickHouse. + +It serve to test ClickHouse schema changes manually and ensure our schema is +robust over time (e.g. when there is millions of events over past months). + diff --git a/cmd/server/README.md b/cmd/server/README.md new file mode 100644 index 0000000..0eb8092 --- /dev/null +++ b/cmd/server/README.md @@ -0,0 +1,10 @@ +# Prisme server + +This directory holds `main` package of Prisme server. + +## Dependency Injection + +Dependencies are injected using [`wire`](https://github.com/google/wire). There, +is a `wire.go` file per Prisme mode (`default` or `ingestion`) in their +respective packages. + diff --git a/config/README.md b/config/README.md new file mode 100644 index 0000000..9be3ab6 --- /dev/null +++ b/config/README.md @@ -0,0 +1,4 @@ +# Config + +Prisme uses environment variables for configuration and this directory holds +simple shell scripts to generate `.env` file. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..6754dbf --- /dev/null +++ b/docs/README.md @@ -0,0 +1,12 @@ +# Documentation + +This directory contains Prisme documentations for maintainer and developer. + +If you're looking for user documentation, checkout our +[website](https://www.prismeanalytics.com/docs). + +Here is a summary of all documentations: +* [User documentation](https://www.prismeanalytics.com/docs): How-to guides, self-host guide and administration documentation. +* [Developer documentation](./dev.md): How to start local development environment and execute tests. +* [Maintainer documentation](./maintainer.md): How to maintain the repository. + diff --git a/docs/dev.md b/docs/dev.md new file mode 100644 index 0000000..11d7973 --- /dev/null +++ b/docs/dev.md @@ -0,0 +1,108 @@ +# Developer documentation + +## Local development + +In order to run Prisme locally, you must have the +[Nix](https://nixos.org/download/) package manager installed. + +Then, you can enter development shell as follow: + +```shell +$ nix develop +``` + +This will start a new shell with all required tools installed (in an isolated +directory named `/nix/store` by default). + +### Build docker image + +Before starting development environment, you must build Prisme docker image at +least once. + +```shell +$ make docker/build +``` + +Prisme uses [Nix](https://nixos.org) to build ultra light docker image that are +**reproducible**. Building twice the image should produce the **exact** same +images. + +Nix builds are slower than `docker build` but don't worry you don't have to +rebuild the entire image at each changes? + +### Start development environment + +You can now start development environment. + +```shell +$ make start +# or +$ make watch/start +``` + +Target `watch/start` will watches all existing Go and docker compose files +and restart the service on changes. + +> **NOTE**: Files created after `watch/start` don't trigger restarts. + +Be sure to read the `Makefile` to read and understand all targets. + +## Documentation + +All packages are documented. You can start a local documentation server using +`godoc -http=:6060`. + +## Testing + +Prisme is designed to be a robust product so we works hard to have good tests +that cover critical parts of the service. This includes but isn't limited to: +* Core features +* Security related features +* Observability (metrics, logs) + +Depending on the feasibility, some tests are written as unit tests, integration +tests or end-to-end tests. + +That being said, we favor end-to-end tests when possible as they're closest +to how Prisme is used in production. + +### Unit tests + +Unit tests follows Go convention and are placed in `*_test.go` files next to +other go files and uses `github.com/stretchr/testify` for asserts. + +You can run unit tests as follows: + +```shell +$ make test/unit +``` + +### Integration tests + +Integration tests are also placed in `*_test.go` files along unit tests but +follows a specific convention: + +```go +// Integration tests function starts with TestInteg. +func TestIntegXXX(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Your test here. +``` + +You can run integration tests as follow. + +```shell +$ make test/integ +``` + +### End-to-end tests + +Finally, end-to-end tests are stored under `tests/`. + +```shell +$ make test/e2e +``` + diff --git a/docs/maintainer.md b/docs/maintainer.md new file mode 100644 index 0000000..edf0394 --- /dev/null +++ b/docs/maintainer.md @@ -0,0 +1,44 @@ +# Maintainer documentation + +This document explains how to maintain the repository. + +## Upgrade dependencies + +It is important to regularly update direct and transitive dependencies to +mitigate security vulnerabilities. + +First, start by listing outdated dependencies: + +```shell +$ go list -u -m all +``` + +Be sure to read all changelogs of a dependency before updating. Then update it: + +```shell +$ go get -u package/path@latest +# or +$ go get -u ./... +``` + +Then, tidy up the `go.mod`: + +```shell +go mod tidy +``` + +Once your done, run all tests to ensure nothing broke and fix it otherwise. + +## Update IP database + +To generate a new IP database, clone +[`negrel/geoacumen-country`](https://github.com/negrel/geoacumen-country) +repository and run `make clean ip2asn-combined.mmdb`. Then copy +`ip2asn-combined.mmdb` to `pkg/embedded/geodb` and commit changes. + +## Release a new version + +Before releasing a new version, be sure to update dependencies, IP database +and maybe Grafana and ClickHouse Docker images to ensure compatibility with +latest versions. + diff --git a/docs/repository.md b/docs/repository.md new file mode 100644 index 0000000..da8b424 --- /dev/null +++ b/docs/repository.md @@ -0,0 +1,41 @@ +# Repository structure + +This document explains repository structure convention. + +``` +$ tree -d --gitignore . +. +├── cmd +│   ├── addevents # CLI utils to add millions fictive events to clickhouse and test clickhouse ingestion performance +│   ├── server # Prisme server main.go +│   └── uaparser # CLI utils to generate test data. +├── config # Directory containing scripts to generate .env config file. +├── deploy # Directory containing deployment files. +├── docs # Documentation. +├── mocks # Mocks holds mocks website used to test manually Prisme when developping. +├── pkg # Prisme Go packages. +├── tests # End-to-end tests. +│   ├── bun # E2E tests using bun. +│   └── perf # E2E benchmarks. +├── tracker # Vanilla JS trackers scripts. +``` + +Note that each directory contains a `README.md` documenting their purposes and +what they should contains. Go packages contains a `doc.go` file instead so +documentation can be read using `godoc -http=:6060` + +## `pkg` - Go packages + +This section details convention about `pkg/` structure. It isn't a documentation +of all packages. + +``` +pkg +├── embedded # Files embedded within Go binary. +├── event # Prisme events structure. +├── handlers # HTTP handlers. +├── middlewares # HTTP middlewares. +├── services # Reusable and swappable features hided behind interfaces. +└── wired # Wire providers of external dependency. +``` + diff --git a/pkg/README.md b/pkg/README.md new file mode 100644 index 0000000..e35d377 --- /dev/null +++ b/pkg/README.md @@ -0,0 +1,15 @@ +# `pkg/` - Go packages + +This document details convention about `pkg/` structure. Packages documentation +is available using `godoc -http=:6060`. + +``` +pkg +├── embedded # Files embedded within Go binary. +├── event # Prisme events structure. +├── handlers # HTTP handlers. +├── middlewares # HTTP middlewares. +├── services # Reusable and swappable features hided behind interfaces. +└── wired # Wire providers of external dependency. +``` + diff --git a/pkg/clickhouse/ch.go b/pkg/clickhouse/ch.go index d14cc5e..c90210b 100644 --- a/pkg/clickhouse/ch.go +++ b/pkg/clickhouse/ch.go @@ -1,3 +1,5 @@ +// Package clickhouse contains wire provider for clickhouse connection and +// migration. package clickhouse import ( @@ -7,6 +9,7 @@ import ( "github.com/rs/zerolog" ) +// Ch define a connection to a ClickHouse instance. type Ch struct { driver.Conn } diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..e0aa6a7 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,2 @@ +// Package config contains utils and structure to load Prisme configurations. +package config diff --git a/pkg/embedded/doc.go b/pkg/embedded/doc.go new file mode 100644 index 0000000..e9c86e4 --- /dev/null +++ b/pkg/embedded/doc.go @@ -0,0 +1,2 @@ +// Package embedded holds files embedded in Go binary. +package embedded diff --git a/pkg/event/doc.go b/pkg/event/doc.go new file mode 100644 index 0000000..51dd698 --- /dev/null +++ b/pkg/event/doc.go @@ -0,0 +1,2 @@ +// Package event holds dumb structure (with no logic) to represent events. +package event diff --git a/pkg/grafana/client.go b/pkg/grafana/client.go index 6ccee11..ecc4efe 100644 --- a/pkg/grafana/client.go +++ b/pkg/grafana/client.go @@ -1,3 +1,5 @@ +// Package grafana holds a generic grafana API client tailored for our specific +// needs. package grafana import ( diff --git a/pkg/handlers/doc.go b/pkg/handlers/doc.go new file mode 100644 index 0000000..03db897 --- /dev/null +++ b/pkg/handlers/doc.go @@ -0,0 +1,2 @@ +// Package handlers holds our Fiber HTTP handlers. +package handlers diff --git a/pkg/log/logger.go b/pkg/log/logger.go index abd12e0..ca50a02 100644 --- a/pkg/log/logger.go +++ b/pkg/log/logger.go @@ -1,3 +1,4 @@ +// Package log holds our zerolog.Logger constructors. package log import ( diff --git a/pkg/middlewares/doc.go b/pkg/middlewares/doc.go new file mode 100644 index 0000000..8d92882 --- /dev/null +++ b/pkg/middlewares/doc.go @@ -0,0 +1,2 @@ +// Package middlewares holds our custom Fiber HTTP middlewares. +package middlewares diff --git a/pkg/secret/secret.go b/pkg/secret/secret.go index 99a5315..1946155 100644 --- a/pkg/secret/secret.go +++ b/pkg/secret/secret.go @@ -1,3 +1,4 @@ +// Package secret contains utils related to secrets. package secret import ( diff --git a/pkg/services/README.md b/pkg/services/README.md new file mode 100644 index 0000000..5bf5660 --- /dev/null +++ b/pkg/services/README.md @@ -0,0 +1,8 @@ +# Services + +This directory holds reusable and swappable features hided behind services +interface. + +Each package/directory should contains a `Service` interface, at least one +implementation and a wire provider for that implementation. + diff --git a/pkg/wired/app.go b/pkg/wired/app.go index 111748d..ecf9aad 100644 --- a/pkg/wired/app.go +++ b/pkg/wired/app.go @@ -9,6 +9,7 @@ import ( "github.com/rs/zerolog" ) +// App holds data used at runtime by main package. type App struct { Config config.Server Fiber *fiber.App diff --git a/pkg/wired/doc.go b/pkg/wired/doc.go new file mode 100644 index 0000000..0ce63ff --- /dev/null +++ b/pkg/wired/doc.go @@ -0,0 +1,3 @@ +// Package wired contains wire provider for external dependencies or code shared +// between Prisme mode. +package wired diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..4804b7c --- /dev/null +++ b/tests/README.md @@ -0,0 +1,3 @@ +# End-to-end tests + +This directory contains end-to-end tests (including benchmarks).