Merge pull request #115 from m-appel/114-add-expire_after-to-cachedse…

…ssion

Add expire_after parameter to CachedSession

config.json.example

@@ -1,4 +1,9 @@
 {
+    "cache": {
+        "directory": "tmp/",
+        "duration_in_days": 6
+    },
+
     "peeringdb": {
         "apikey": ""
     },

iyp/crawlers/peeringdb/fac.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import sys
+from datetime import timedelta
 
 import flatdict
 import iso3166
@@ -24,16 +25,23 @@
 FACID_LABEL = 'PeeringdbFacID'
 
 API_KEY = ''
+CACHE_DIR = ''
+CACHE_DURATION = requests_cache.DO_NOT_CACHE
 if os.path.exists('config.json'):
-    API_KEY = json.load(open('config.json', 'r'))['peeringdb']['apikey']
+    with open('config.json', 'r') as f:
+        config = json.load(f)
+    API_KEY = config['peeringdb']['apikey']
+    CACHE_DIR = config['cache']['directory']
+    CACHE_DURATION = timedelta(days=config['cache']['duration_in_days'])
+    del config  # Do not leave as a global variable.
 
 
 class Crawler(BaseCrawler):
     def __init__(self, organization, url, name):
         """Initialisation for pushing peeringDB facilities to IYP."""
 
         self.headers = {'Authorization': 'Api-Key ' + API_KEY}
-        self.requests = requests_cache.CachedSession(ORG)
+        self.requests = requests_cache.CachedSession(os.path.join(CACHE_DIR, ORG), expire_after=CACHE_DURATION)
 
         super().__init__(organization, url, name)
 

iyp/crawlers/peeringdb/ix.py

@@ -3,7 +3,7 @@
 import logging
 import os
 import sys
-from datetime import datetime, time, timezone
+from datetime import datetime, time, timedelta, timezone
 
 import flatdict
 import requests_cache
@@ -34,8 +34,15 @@
 FACID_LABEL = 'PeeringdbFacID'
 
 API_KEY = ''
+CACHE_DIR = ''
+CACHE_DURATION = requests_cache.DO_NOT_CACHE
 if os.path.exists('config.json'):
-    API_KEY = json.load(open('config.json', 'r'))['peeringdb']['apikey']
+    with open('config.json', 'r') as f:
+        config = json.load(f)
+    API_KEY = config['peeringdb']['apikey']
+    CACHE_DIR = config['cache']['directory']
+    CACHE_DURATION = timedelta(days=config['cache']['duration_in_days'])
+    del config  # Do not leave as a global variable.
 
 
 def handle_social_media(d: dict, website_set: set = None):
@@ -82,7 +89,7 @@ def __init__(self, organization, url, name):
         self.nets = {}
 
         # Using cached queries
-        self.requests = requests_cache.CachedSession(ORG)
+        self.requests = requests_cache.CachedSession(os.path.join(CACHE_DIR, ORG), expire_after=CACHE_DURATION)
 
         # connection to IYP database
         super().__init__(organization, url, name)

iyp/crawlers/peeringdb/org.py

@@ -3,6 +3,7 @@
 import logging
 import os
 import sys
+from datetime import timedelta
 
 import flatdict
 import iso3166
@@ -21,16 +22,23 @@
 ORGID_LABEL = 'PeeringdbOrgID'
 
 API_KEY = ''
+CACHE_DIR = ''
+CACHE_DURATION = requests_cache.DO_NOT_CACHE
 if os.path.exists('config.json'):
-    API_KEY = json.load(open('config.json', 'r'))['peeringdb']['apikey']
+    with open('config.json', 'r') as f:
+        config = json.load(f)
+    API_KEY = config['peeringdb']['apikey']
+    CACHE_DIR = config['cache']['directory']
+    CACHE_DURATION = timedelta(days=config['cache']['duration_in_days'])
+    del config  # Do not leave as a global variable.
 
 
 class Crawler(BaseCrawler):
     def __init__(self, organization, url, name):
         """Initialisation for pushing peeringDB organizations to IYP."""
 
         self.headers = {'Authorization': 'Api-Key ' + API_KEY}
-        self.requests = requests_cache.CachedSession(ORG)
+        self.requests = requests_cache.CachedSession(os.path.join(CACHE_DIR, ORG), expire_after=CACHE_DURATION)
 
         super().__init__(organization, url, name)