improve logging, pin py2neo>=2020

.gitignore

@@ -232,3 +232,4 @@ dmypy.json
 .pyre/
 
 # End of https://www.gitignore.io/api/python
+.env

covid_graph/helper.py

@@ -3,7 +3,7 @@
 import zipfile
 import logging
 
-from py2neo.database import ClientError
+from py2neo import ClientError
 
 log = logging.getLogger(__name__)
 
@@ -23,10 +23,12 @@ def setup_db(graph):
     ]
 
     for index in indexes:
+        log.info(f"Try to create index {index[0]}: {index[1]}")
         try:
             graph.schema.create_index(index[0], index[1])
-        except ClientError:
-            pass
+        except ClientError as e:
+            log.info(f"Cannot create index {index[0]}: {index[1]}, pass and continue")
+            log.error(e)
 
 
 def unzip_file(zip_file_path, skip_existing=False):

docker-compose.yml

@@ -4,8 +4,6 @@ services:
   run_data_jhu_population:
     build: .
     environment:
-      - GC_NEO4J_URL=bolt://host.docker.internal:7687
-      - GC_NEO4J_USER=neo4j
-      - GC_NEO4J_PASSWORD=test
       - RUN_MODE=prod
+      - NEO4J=${NEO4J}
     #command: python run.py

requirements.txt

@@ -1,6 +1,4 @@
-neo4j
-py2neo
-#git+https://github.com/technige/py2neo.git@v5#egg=py2neo
+py2neo>=2020.0.0
 requests
 python-dateutil>=2.8.1
 graphio>=0.0.12

run.py

@@ -1,48 +1,48 @@
 import os
-import sys
 import logging
 import py2neo
-
+import json
 
 logging.basicConfig(level=logging.DEBUG)
-logging.getLogger('py2neo.connect.bolt').setLevel(logging.WARNING)
-logging.getLogger('py2neo.connect').setLevel(logging.WARNING)
+logging.getLogger('py2neo.client.bolt').setLevel(logging.WARNING)
+logging.getLogger('py2neo.client').setLevel(logging.WARNING)
+logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING)
 logging.getLogger('graphio').setLevel(logging.WARNING)
-logging.getLogger('neobolt').setLevel(logging.WARNING)
 
 log = logging.getLogger(__name__)
 
 # import and setup
 from covid_graph import helper, post, jhu, unwpp
 
 ROOT_DIR = os.getenv('ROOT_DIR', '/download')
-GC_NEO4J_URL = os.getenv('GC_NEO4J_URL', 'bolt://localhost:7687')
-GC_NEO4J_USER = os.getenv('GC_NEO4J_USER', 'neo4j')
-GC_NEO4J_PASSWORD = os.getenv('GC_NEO4J_PASSWORD', 'test')
 RUN_MODE = os.getenv('RUN_MODE', 'prod')
 
+NEO4J_CONFIG_STRING = os.getenv("NEO4J")
+
+try:
+    log.info(NEO4J_CONFIG_STRING)
+    NEO4J_CONFIG_DICT = json.loads(NEO4J_CONFIG_STRING)
+except json.decoder.JSONDecodeError:
+    # try to replace single quotes with double quotes
+    # JSON always expects double quotes, common mistake when writing JSON strings
+    NEO4J_CONFIG_STRING = NEO4J_CONFIG_STRING.replace("'", '"')
+    log.info(NEO4J_CONFIG_STRING)
+    NEO4J_CONFIG_DICT = json.loads(NEO4J_CONFIG_STRING)
+
 if RUN_MODE.lower() == 'test':
     import pytest
+
     log.info("Run tests")
     pytest.main()
 
 else:
-    for v in [ROOT_DIR, GC_NEO4J_URL, GC_NEO4J_USER, GC_NEO4J_PASSWORD]:
-        log.debug(v)
 
-    graph = py2neo.Graph(GC_NEO4J_URL, user=GC_NEO4J_USER, password=GC_NEO4J_PASSWORD)
-    log.debug(graph)
-
-    result = list(graph.run("MATCH (a) RETURN a LIMIT 1"))
-    log.debug(result)
+    graph = py2neo.Graph(host=NEO4J_CONFIG_DICT['host'], user=NEO4J_CONFIG_DICT['user'],
+                         password=NEO4J_CONFIG_DICT['password'], secure=NEO4J_CONFIG_DICT['secure'], verify=False)
 
     # setup DB
     helper.setup_db(graph)
 
-    if not os.path.exists(ROOT_DIR):
-        os.makedirs(ROOT_DIR)
-    ###
-
     # download data
     jhu_zip_file = jhu.download_jhu(ROOT_DIR)
     jhu_dir = helper.unzip_file(jhu_zip_file)