diff --git a/iyp/crawlers/caida/asrank.py b/iyp/crawlers/caida/asrank.py index 6d797eb..d9e9cd2 100644 --- a/iyp/crawlers/caida/asrank.py +++ b/iyp/crawlers/caida/asrank.py @@ -3,6 +3,7 @@ import logging import os import sys +from datetime import datetime, timezone import flatdict import requests @@ -20,6 +21,16 @@ def __init__(self, organization, url, name): super().__init__(organization, url, name) self.reference['reference_url_info'] = 'https://asrank.caida.org/' + def __set_modification_time(self): + try: + date = requests.get('https://api.asrank.caida.org/v2/restful/datasets').json()['data'][0]['date'] + self.reference['reference_time_modification'] = datetime.strptime(date, + '%Y-%m-%d').replace(tzinfo=timezone.utc) + logging.info(f'Dataset modification time: {date}') + except Exception as e: + logging.warning(f'Failed to set modification time: {e}') + return + def run(self): """Fetch networks information from ASRank and push to IYP.""" nodes = list() @@ -42,6 +53,7 @@ def run(self): nodes += ranking['edges'] logging.info(f'Fetched {len(nodes):,d} ranks.') + self.__set_modification_time() # Collect all ASNs, names, and countries asns = set()