Merge branch 'main' into manrs-performance-improvement

.github/workflows/pre-commit.yml

@@ -0,0 +1,22 @@
+name: Pre-commit CI
+
+on: [push, pull_request]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version-file: '.python-version'
+        cache: 'pip'
+
+    - name: Install dependencies
+      run: pip install -r requirements.txt
+
+    - name: pre-commit
+      uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -1,16 +1,17 @@
 files: \.py$
 repos:
 -   repo: https://github.com/PyCQA/autoflake
-    rev: v2.2.0
+    rev: v2.2.1
     hooks:
     -   id: autoflake
         args: [--in-place]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
     -   id: isort
+        args: [--thirdparty, neo4j]
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.2
+    rev: v2.0.4
     hooks:
     -   id: autopep8
 - repo: https://github.com/PyCQA/docformatter
@@ -19,12 +20,12 @@ repos:
     - id: docformatter
       args: [--in-place, --wrap-summaries, '88', --wrap-descriptions, '88']
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: double-quote-string-fixer
     -   id: mixed-line-ending
         args: [--fix, lf]
 -   repo: https://github.com/PyCQA/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     -   id: flake8

config.json.example

@@ -69,6 +69,7 @@
             "iyp.crawlers.pch.daily_routing_snapshots_v6",
             "iyp.crawlers.emileaben.as_names",
             "iyp.crawlers.ripe.atlas_probes",
+            "iyp.crawlers.ripe.atlas_measurements",
             "iyp.crawlers.iana.root_zone",
             "iyp.crawlers.alice_lg.amsix",
             "iyp.crawlers.alice_lg.bcix",

create_db.py

@@ -126,9 +126,9 @@ def __init__(self, message):
         crawler.close()
         if not relations_count_new > relations_count:
             error_message = (
-                f"Unexpected relation count change in the crawler '{name}': "
-                f"Expected new relations ({relations_count_new}) "
-                f"to be greater than the previous relations ({relations_count})."
+                f'Unexpected relation count change in the crawler "{name}": '
+                f'Expected new relations ({relations_count_new}) '
+                f'to be greater than the previous relations ({relations_count}).'
             )
             raise RelationCountError(error_message)
         status[module_name] = 'OK'

iyp/__init__.py

@@ -3,11 +3,11 @@
 import logging
 import os
 import pickle
-import sys
 from datetime import datetime, time, timezone
 from shutil import rmtree
 from typing import Optional
 
+import requests
 from neo4j import GraphDatabase
 
 BATCH_SIZE = 50000
@@ -65,18 +65,48 @@ def dict2str(d, eq=':', pfx=''):
     for key, value in d.items():
         if isinstance(value, str) and '"' in value:
             escaped = value.replace("'", r"\'")
-            data.append(f"{pfx+key}{eq} '{escaped}'")
+            data.append(f"{pfx + key}{eq} '{escaped}'")
         elif isinstance(value, str) or isinstance(value, datetime):
-            data.append(f'{pfx+key}{eq} "{value}"')
+            data.append(f'{pfx + key}{eq} "{value}"')
         elif value is None:
             # Neo4j does not have the concept of empty properties.
             pass
         else:
-            data.append(f'{pfx+key}{eq} {value}')
+            data.append(f'{pfx + key}{eq} {value}')
 
     return '{' + ','.join(data) + '}'
 
 
+class RequestStatusError(requests.HTTPError):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class JSONDecodeError(ValueError):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class MissingKeyError(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class ConnectionError(requests.exceptions.ConnectionError):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class AddressValueError(ValueError):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
 class IYP(object):
 
     def __init__(self):
@@ -95,7 +125,7 @@ def __init__(self):
         self.db = GraphDatabase.driver(uri, auth=(self.login, self.password))
 
         if self.db is None:
-            sys.exit('Could not connect to the Neo4j database!')
+            raise ConnectionError('Could not connect to the Neo4j database!')
         # Raises an exception if there is a problem.
         # "Best practice" is to just let the program
         # crash: https://neo4j.com/docs/python-manual/current/connect/

iyp/crawlers/apnic/eyeball.py

@@ -6,7 +6,7 @@
 import iso3166
 import requests
 
-from iyp import BaseCrawler
+from iyp import BaseCrawler, RequestStatusError
 
 # URL to APNIC API
 URL = 'http://v6data.data.labs.apnic.net/ipv6-measurement/Economies/'
@@ -40,7 +40,7 @@ def run(self):
             self.url = URL + f'{cc}/{cc}.asns.json?m={MIN_POP_PERC}'
             req = requests.get(self.url)
             if req.status_code != 200:
-                sys.exit('Error while fetching data for ' + cc)
+                raise RequestStatusError(f'Error while fetching data for {cc}')
 
             asns = set()
             names = set()