add workflows

.github/workflows/data-validation.yml

@@ -0,0 +1,29 @@
+name: databuilder
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 0"
+
+jobs:
+  run-workflow-script:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v2
+        with:
+          ref: main
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.11.4"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run Python script
+        run: python src/validate_data.py

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
           pip install -r requirements.txt
 
       - name: Run Python script
-        run: python src/workflow.py
+        run: python src/combine_data.py
 
       - name: Archive generated files as artifact
         uses: actions/upload-artifact@v2

README.md

@@ -34,6 +34,4 @@ url = "https://example2.url"
 
 # TODO:
 
-- validate data links to not return 404
 - give warnings using PRs or Discord hook?
--

requirements.txt

@@ -1,2 +1,3 @@
 numpy==1.24.4
-pydantic==2.7.4
+pydantic==2.8.2
+Requests==2.32.3

src/combine_data.py

@@ -0,0 +1,11 @@
+from settings import ROOT_DIRECTORY, OUTPUT_DIRECTORY
+from tasks.export_stands import export_hp_csv
+from tasks.read_grp_data import GrpDataReader
+from tasks.toml_data import TomlData
+
+
+if __name__ == "__main__":
+    airports = GrpDataReader().process_stand_files(ROOT_DIRECTORY)
+    export_hp_csv(airports, OUTPUT_DIRECTORY)
+
+    TomlData(data_dir=ROOT_DIRECTORY, output_dir=OUTPUT_DIRECTORY)

src/settings.py

@@ -0,0 +1,2 @@
+ROOT_DIRECTORY = "data/"
+OUTPUT_DIRECTORY = "api/"

src/tasks/toml_data.py

@@ -0,0 +1,93 @@
+import json
+import os
+import sys
+import tomllib
+import requests
+from typing import List
+
+from views.airport import Airport, AirportData
+
+
+class TomlData:
+    def __init__(self, data_dir: str, output_dir: str, export: bool = True):
+        self.data_dir = data_dir
+        self.output_dir = output_dir
+        self.data = AirportData(airports=[])
+
+        self.errors = []
+
+        self.load_toml_data()
+        self.validate_data()
+
+        self.check_errors()
+
+        if export:
+            self.export_data_json()
+
+    def load_toml_data(self):
+        """Loads the TOML data from the data directory."""
+        for root, _, files in os.walk(self.data_dir):
+            for file in files:
+                if file.endswith(".toml"):
+                    file_path = os.path.join(root, file)
+                    self.process_toml(file_path)
+
+    def process_toml(self, file_path: str):
+        """Processes a single TOML file and adds its content to self.data as an Airport instance."""
+        try:
+            with open(file_path, "rb") as f:
+                toml_content = tomllib.load(f)
+
+            if "airport" in toml_content:
+                for airport_data in toml_content["airport"]:
+                    airport = Airport(**airport_data)
+                    self.data.airports.append(airport)
+            else:
+                print(f"Warning: 'airport' key not found in {file_path}")
+
+        except Exception as e:
+            self.errors.append(f"Failed to process {file_path}: {e}")
+
+    def validate_url(self, url: str):
+        try:
+            response = requests.head(url, allow_redirects=True, timeout=2)
+            print(f"Checked {url}, status code: {response.status_code}")
+            return response.status_code != 404
+        except requests.exceptions.RequestException as e:
+            # If there was an issue with the request, return False
+            print(f"Error checking {url}: {e}")
+            return False
+
+    def validate_data(self):
+        for element in self.data.airports:
+            for link in element.links:
+                url_is_valid = self.validate_url(link.url)
+                if not url_is_valid:
+                    self.errors.append(
+                        f"Airport {element.icao} has an invalid URL, see: {link}"
+                    )
+
+    def check_errors(self):
+        if len(self.errors) == 0:
+            return
+
+        for error in self.errors:
+            print(error)
+
+        sys.exit(1)
+
+    def export_data_json(self):
+        if len(self.errors) != 0:
+            return
+
+        try:
+            json_data = self.data.model_dump(mode="json")
+            output_path = os.path.join(self.output_dir, "airports.json")
+
+            with open(output_path, "w") as json_file:
+                json.dump(json_data, json_file, indent=4)
+            print(f"Data successfully exported to {output_path}")
+
+        except Exception as e:
+            print(f"Failed to export data to JSON: {e}")
+            sys.exit(1)

src/validate_data.py

@@ -0,0 +1,6 @@
+from settings import ROOT_DIRECTORY, OUTPUT_DIRECTORY
+from tasks.toml_data import TomlData
+
+
+if __name__ == "__main__":
+    TomlData(data_dir=ROOT_DIRECTORY, output_dir=OUTPUT_DIRECTORY, export=False)

src/views/airport.py

@@ -0,0 +1,17 @@
+from typing import List, Literal, Optional
+from pydantic import BaseModel, HttpUrl
+
+
+class Link(BaseModel):
+    category: Optional[Literal["Scenery", "Charts", "Briefing"]] = None
+    name: str
+    url: HttpUrl
+
+
+class Airport(BaseModel):
+    icao: str
+    links: List[Link]
+
+
+class AirportData(BaseModel):
+    airports: List[Airport]