diff --git a/.gitignore b/.gitignore index 18f3946..a80da0f 100644 --- a/.gitignore +++ b/.gitignore @@ -232,3 +232,4 @@ dmypy.json .pyre/ # End of https://www.gitignore.io/api/python +.env diff --git a/covid_graph/helper.py b/covid_graph/helper.py index bd9f874..19b3586 100644 --- a/covid_graph/helper.py +++ b/covid_graph/helper.py @@ -3,7 +3,7 @@ import zipfile import logging -from py2neo.database import ClientError +from py2neo import ClientError log = logging.getLogger(__name__) @@ -23,10 +23,12 @@ def setup_db(graph): ] for index in indexes: + log.info(f"Try to create index {index[0]}: {index[1]}") try: graph.schema.create_index(index[0], index[1]) - except ClientError: - pass + except ClientError as e: + log.info(f"Cannot create index {index[0]}: {index[1]}, pass and continue") + log.error(e) def unzip_file(zip_file_path, skip_existing=False): diff --git a/docker-compose.yml b/docker-compose.yml index bf3ddb6..1999ae9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,6 @@ services: run_data_jhu_population: build: . environment: - - GC_NEO4J_URL=bolt://host.docker.internal:7687 - - GC_NEO4J_USER=neo4j - - GC_NEO4J_PASSWORD=test - RUN_MODE=prod + - NEO4J=${NEO4J} #command: python run.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 392828c..b472d9c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ -neo4j -py2neo -#git+https://github.com/technige/py2neo.git@v5#egg=py2neo +py2neo>=2020.0.0 requests python-dateutil>=2.8.1 graphio>=0.0.12 diff --git a/run.py b/run.py index eebfe77..d2cc19d 100644 --- a/run.py +++ b/run.py @@ -1,14 +1,13 @@ import os -import sys import logging import py2neo - +import json logging.basicConfig(level=logging.DEBUG) -logging.getLogger('py2neo.connect.bolt').setLevel(logging.WARNING) -logging.getLogger('py2neo.connect').setLevel(logging.WARNING) +logging.getLogger('py2neo.client.bolt').setLevel(logging.WARNING) +logging.getLogger('py2neo.client').setLevel(logging.WARNING) +logging.getLogger('urllib3.connectionpool').setLevel(logging.WARNING) logging.getLogger('graphio').setLevel(logging.WARNING) -logging.getLogger('neobolt').setLevel(logging.WARNING) log = logging.getLogger(__name__) @@ -16,33 +15,34 @@ from covid_graph import helper, post, jhu, unwpp ROOT_DIR = os.getenv('ROOT_DIR', '/download') -GC_NEO4J_URL = os.getenv('GC_NEO4J_URL', 'bolt://localhost:7687') -GC_NEO4J_USER = os.getenv('GC_NEO4J_USER', 'neo4j') -GC_NEO4J_PASSWORD = os.getenv('GC_NEO4J_PASSWORD', 'test') RUN_MODE = os.getenv('RUN_MODE', 'prod') +NEO4J_CONFIG_STRING = os.getenv("NEO4J") + +try: + log.info(NEO4J_CONFIG_STRING) + NEO4J_CONFIG_DICT = json.loads(NEO4J_CONFIG_STRING) +except json.decoder.JSONDecodeError: + # try to replace single quotes with double quotes + # JSON always expects double quotes, common mistake when writing JSON strings + NEO4J_CONFIG_STRING = NEO4J_CONFIG_STRING.replace("'", '"') + log.info(NEO4J_CONFIG_STRING) + NEO4J_CONFIG_DICT = json.loads(NEO4J_CONFIG_STRING) + if RUN_MODE.lower() == 'test': import pytest + log.info("Run tests") pytest.main() else: - for v in [ROOT_DIR, GC_NEO4J_URL, GC_NEO4J_USER, GC_NEO4J_PASSWORD]: - log.debug(v) - graph = py2neo.Graph(GC_NEO4J_URL, user=GC_NEO4J_USER, password=GC_NEO4J_PASSWORD) - log.debug(graph) - - result = list(graph.run("MATCH (a) RETURN a LIMIT 1")) - log.debug(result) + graph = py2neo.Graph(host=NEO4J_CONFIG_DICT['host'], user=NEO4J_CONFIG_DICT['user'], + password=NEO4J_CONFIG_DICT['password'], secure=NEO4J_CONFIG_DICT['secure'], verify=False) # setup DB helper.setup_db(graph) - if not os.path.exists(ROOT_DIR): - os.makedirs(ROOT_DIR) - ### - # download data jhu_zip_file = jhu.download_jhu(ROOT_DIR) jhu_dir = helper.unzip_file(jhu_zip_file)