test: ✅ add pkg testing and action

.github/workflows/test.yml

@@ -0,0 +1,25 @@
+name: Go Test
+
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ^1.22
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@v2
+    - name: Install templ
+      run: go install github.com/a-h/templ/cmd/templ@v0
+    - name: Generate template code
+      run: templ generate
+    - name: Get dependencies
+      run: go mod download
+    - name: Test
+      run: SQL_PATH="memory" ROOT=${GITHUB_WORKSPACE} go test -v ./...

README.md

@@ -67,6 +67,14 @@ Running the mutate script will fill the database with the latest backbone taxono
 go run ./scripts/mutate/mutate.go
 ```
 
+### Testing
+
+To run the tests you will need to set the `SQL_PATH` and `ROOT` environment variables. The `SQL_PATH` is the path to the database file (from the root) and `ROOT` is the path to the root of the project.
+
+```bash
+SQL_PATH="memory" ROOT=/gbif-extinct go test -v ./...
+```
+
 ### Docker
 
 GitHub action is used to generate the web-sever as a docker container [hub.docker.com/r/hannesoberreiter/gbif-extinct](https://hub.docker.com/r/hannesoberreiter/gbif-extinct). See the [Dockerfile](Dockerfile) for details of the build and the [docker-compose.yml](docker-compose.yml) for the deployment.

internal/internal.go

@@ -15,14 +15,15 @@ var (
 )
 
 type config struct {
+	ROOT               string `mapstructure:"ROOT"`
 	SqlPath            string `mapstructure:"SQL_PATH"`
 	TaxonBackbonePath  string `mapstructure:"TAXON_BACKBONE_PATH"`
 	TaxonSimplePath    string `mapstructure:"TAXON_SIMPLE_PATH"`
 	UserAgentPrefix    string `mapstructure:"USER_AGENT_PREFIX"`
 	CronJobIntervalSec int    `mapstructure:"CRON_JOB_INTERVAL_SEC"`
 }
 
-func init() {
+func Load() {
 	slog.Debug("Initializing internal package")
 	loadEnv()
 	loadDb()
@@ -33,25 +34,34 @@ func init() {
 // therefore as long as the server is running we cannot connect to the database externally
 func loadDb() {
 	var err error
-	DB, err = sql.Open("duckdb", Config.SqlPath)
+	var dbPath string
+	if Config.SqlPath == "memory" {
+		slog.Info("No database path set, using in-memory database")
+		dbPath = ""
+	} else {
+		dbPath = Config.ROOT + Config.SqlPath
+	}
+
+	DB, err = sql.Open("duckdb", dbPath)
 	if err != nil {
-		slog.Debug("Failed to connect to database.", "path", Config.SqlPath)
+		slog.Debug("Failed to connect to database.", "path", dbPath)
 		log.Fatal(err)
 	}
-	slog.Info("Connected to database.", "path", Config.SqlPath)
+	slog.Info("Connected to database.", "path", dbPath)
 }
 
 // loadEnv loads the environment variables from the .env file or the system environment
 // most have sane defaults anyway.
 // The order is as follows default < .env < system environment.
 func loadEnv() {
-	slog.Debug("Loading enviroment variables")
+	slog.Debug("Loading environment variables")
 
-	viper.SetDefault("SQL_PATH", "./db/duck.db")
-	viper.SetDefault("TAXON_BACKBONE_PATH", "./Taxon.tsv")
-	viper.SetDefault("TAXON_SIMPLE_PATH", "./simple.txt")
+	viper.SetDefault("SQL_PATH", "/db/duck.db")
+	viper.SetDefault("TAXON_BACKBONE_PATH", "/Taxon.tsv")
+	viper.SetDefault("TAXON_SIMPLE_PATH", "/simple.txt")
 	viper.SetDefault("USER_AGENT_PREFIX", "local")
-	viper.SetDefault("CRON_JOB_INTERVAL_SEC", 60)
+	viper.SetDefault("CRON_JOB_INTERVAL_SEC", 0)
+	viper.SetDefault("ROOT", ".")
 
 	viper.SetConfigName(".env")
 	viper.SetConfigType("env")
@@ -60,7 +70,7 @@ func loadEnv() {
 
 	err := viper.ReadInConfig()
 	if err != nil {
-		slog.Error("Error loading .env file", "error", err)
+		slog.Warn("Error loading .env file", "error", err)
 	}
 
 	err = viper.Unmarshal(&Config)

internal/migrate.go

@@ -2,16 +2,20 @@ package internal
 
 import (
 	"context"
+	"database/sql"
 	"log"
 	"log/slog"
 	"os"
 )
 
-const migrationsDir = "./migrations"
+var migrationsDir = "./migrations"
 
 // Helper function to run migration files. Its pretty simple and the migration will always run from the beginning.
 // The migrations must be placed in "./migrations" and have the file extension ".sql".
-func Migrations() {
+func Migrations(db *sql.DB, migrationsPath string) {
+	if migrationsPath != "" {
+		migrationsDir = migrationsPath + "/migrations"
+	}
 	slog.Debug("Running migrations")
 	fs, err := os.ReadDir(migrationsDir)
 	if err != nil {
@@ -26,14 +30,14 @@ func Migrations() {
 			continue
 		}
 		slog.Info("Found migration file", "file", f.Name())
-		file, err := os.ReadFile("migrations/" + f.Name())
+		file, err := os.ReadFile(migrationsDir + "/" + f.Name())
 		if err != nil {
 			log.Fatal(err)
 		}
 		queries = append(queries, string(file))
 	}
 	ctx := context.Background()
-	conn, err := DB.Conn(ctx)
+	conn, err := db.Conn(ctx)
 	if err != nil {
 		slog.Error("Failed to create migration connection", err)
 		return

pkg/gbif/gbif_test.go

@@ -0,0 +1,171 @@
+package gbif
+
+import (
+	"context"
+	"database/sql"
+	"log"
+	"log/slog"
+	"strings"
+	"testing"
+
+	"github.com/HannesOberreiter/gbif-extinct/internal"
+)
+
+// Demo data for testing, it is no synonym
+var DemoTaxa = []string{
+	"4492208", "4492208", "'Urocerus gigas'", "'Animalia'", "'Arthropoda'", "'Insecta'", "'Hymenoptera'", "'Siricidae'", "'Urocerus'",
+}
+
+// Demo data for testing which is a synonym
+var DemoSyn = []string{
+	"8071112", "4492208", "'Urocerus gigas'", "'Ichneumon gigas'", "'Animalia'", "'Arthropoda'", "'Insecta'", "'Hymenoptera'", "'Siricidae'", "'Urocerus'", "true",
+}
+
+func TestUpdateConfig(t *testing.T) {
+	UpdateConfig(Config{UserAgentPrefix: "test"})
+}
+
+func TestFetchLatest(t *testing.T) {
+	loadDemo()
+	/* Endemic species to Austria, fast response low number of results */
+	/* https://www.gbif.org/species/4560445 */
+	var id = "4560445"
+	res := FetchLatest(id)
+	if res == nil {
+		t.Errorf("got %v, wanted %v", res, "not nil")
+	}
+	if len(res) == 0 {
+		t.Errorf("got %d, wanted < %d", len(res), 1)
+	}
+	if res[0].TaxonID != id {
+		t.Errorf("got %s, wanted %s", res[0].TaxonID, id)
+	}
+
+	res = FetchLatest("123456")
+	if res != nil {
+		t.Errorf("got %v, wanted %v", res, nil)
+	}
+}
+
+func TestSaveObservations(t *testing.T) {
+	loadDemo()
+	observation := LatestObservation{
+		TaxonID:                 DemoTaxa[0],
+		ObservationID:           "123456",
+		ObservationOriginalDate: "1989-01-05",
+		ObservationDate:         "1989-01-05",
+		CountryCode:             "AT",
+	}
+
+	var observations [][]LatestObservation
+	observations = append(observations, []LatestObservation{observation})
+
+	ctx := context.Background()
+	conn, err := internal.DB.Conn(ctx)
+	if err != nil {
+		slog.Error("Failed to create connection", err)
+	}
+	defer conn.Close()
+
+	SaveObservation(observations, conn, ctx)
+
+	var count int
+	err = internal.DB.QueryRow("SELECT COUNT(*) FROM observations WHERE TaxonID = ?", DemoTaxa[0]).Scan(&count)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if count != 1 {
+		t.Errorf("got %d, wanted %d", count, 1)
+	}
+}
+
+func TestGetOutdatedObservations(t *testing.T) {
+	loadDemo()
+	want := GetOutdatedObservations(internal.DB)
+	if len(want) != 1 {
+		t.Errorf("got %d, wanted %d", len(want), 1)
+	}
+	if want[0] != DemoTaxa[0] {
+		t.Errorf("got %s, wanted %s", want[0], DemoTaxa[0])
+	}
+}
+
+func TestUpdateLastFetchStatus(t *testing.T) {
+	loadDemo()
+	var lastFetch sql.NullTime
+
+	/* If not set for a taxa return null */
+	err := internal.DB.QueryRow("SELECT LastFetch FROM taxa WHERE TaxonID = ?", DemoTaxa[0]).Scan(&lastFetch)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if lastFetch.Valid {
+		t.Errorf("got %v, wanted %v", lastFetch.Valid, false)
+	}
+
+	/* Update last fetch status and it should return one */
+	UpdateLastFetchStatus(internal.DB, DemoTaxa[0])
+
+	err = internal.DB.QueryRow("SELECT LastFetch FROM taxa WHERE TaxonID = ?", DemoTaxa[0]).Scan(&lastFetch)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if !lastFetch.Valid {
+		t.Errorf("got %v, wanted %v", lastFetch.Valid, true)
+	}
+	if lastFetch.Time.IsZero() {
+		t.Errorf("got %v, wanted %v", lastFetch.Time.IsZero(), false)
+	}
+}
+
+func TestGetSynonymID(t *testing.T) {
+	loadDemo()
+
+	/* Actual synonym */
+	want, err := GetSynonymID(internal.DB, DemoSyn[0])
+	if err != nil {
+		t.Errorf("got %v, wanted %v", err, nil)
+	}
+	if want != DemoSyn[1] {
+		t.Errorf("got %s, wanted %s", want, DemoSyn[1])
+	}
+
+	/* If no synonym return itself */
+	want, err = GetSynonymID(internal.DB, DemoTaxa[0])
+	if err != nil {
+		t.Errorf("got %v, wanted %v", err, nil)
+	}
+	if want != DemoTaxa[0] {
+		t.Errorf("got %s, wanted %s", want, DemoTaxa[0])
+	}
+
+	/* If no taxon in database return error */
+	_, err = GetSynonymID(internal.DB, "123456")
+	if err == nil {
+		t.Errorf("got %v, wanted %v", err, "error")
+	}
+}
+
+// Helper to setup memory database and data
+func loadDemo() {
+	slog.SetLogLoggerLevel(slog.LevelError)
+	internal.Load()
+	internal.Migrations(internal.DB, internal.Config.ROOT)
+
+	_, err := internal.DB.Exec(`
+		INSERT OR REPLACE INTO taxa
+		(TaxonID, SynonymID, ScientificName, TaxonKingdom, TaxonPhylum, TaxonClass, TaxonOrder, TaxonFamily, TaxonGenus)
+		VALUES (` + strings.Join(DemoTaxa, ",") + ")")
+	if err != nil {
+		slog.Error("Database error", err)
+		log.Fatal(err)
+	}
+	_, err = internal.DB.Exec(`
+		INSERT OR REPLACE INTO taxa
+		(TaxonID, SynonymID, SynonymName, ScientificName, TaxonKingdom, TaxonPhylum, TaxonClass, TaxonOrder, TaxonFamily, TaxonGenus, isSynonym)
+		VALUES  (` + strings.Join(DemoSyn, ",") + ")")
+	if err != nil {
+		slog.Error("Database error", err)
+		log.Fatal(err)
+	}
+}