Update/cli/carrot mapper

carrot/cli/cli.py

@@ -10,10 +10,12 @@
 from .subcommands.pseudonymise import pseudonymise
 
 from carrot.tools.logger import _Logger as Logger
-import carrot as c
 
+import carrot as c
 import click
 import json
+from dotenv import dotenv_values
+
 
 
 @click.group(invoke_without_command=True)
@@ -22,17 +24,38 @@
               type=click.Choice(['0','1','2','3']),
               default='2',
               help="change the level for log messaging. 0 - ERROR, 1 - WARNING, 2 - INFO (default), 3 - DEBUG ")
+@click.option("token","--carrot-mapper-token",help="specify a token for interacting with the CaRROT-Mapper tool",default=None)
+@click.option("url","--carrot-mapper-url",help="specify a url for a CaRROT-Mapper tool instance",default=None)
 @click.option("-cp", "--cprofile", is_flag=True, help='use cProfile to profile the tool')
 @click.pass_context
-def carrot(ctx,version,log_level,cprofile):
+def carrot(ctx,version,log_level,cprofile,token,url):
+
     if ctx.invoked_subcommand == None :
         if version:
             click.echo(c.__version__)
         else:
             click.echo(ctx.get_help()) 
         return
-
 
+    #load any variables/value stored in a local .env file
+    env_values = dotenv_values(".env")
+    token = (
+        env_values['CARROT_MAPPER_TOKEN']
+        if 'CARROT_MAPPER_TOKEN' in env_values else None
+    ) if token is None else token
+
+    url = (
+        env_values['CARROT_MAPPER_URL']
+        if 'CARROT_MAPPER_URL' in env_values else None
+    ) if url is None else url
+
+    ctx.obj = {
+        'token': token,
+        'url': url
+    }
+
+    ctx.obj['headers'] = c.tools.get_request_headers(token) if token else None
+
     c.params['debug_level'] = int(log_level)
     log = Logger("carrot")
     if cprofile:

carrot/cli/subcommands/generate.py

@@ -10,6 +10,7 @@
 import random
 import datetime
 import time
+from dotenv import dotenv_values
 
 class MissingToken(Exception):
     pass
@@ -23,33 +24,21 @@ def generate():
 def synthetic():
     pass
 
-@click.command(help="generate synthetic data from a ScanReport ID from CCOM")
+@click.command(help="generate synthetic data from a ScanReport ID from CaRROT-Mapper website")
 @click.option("-i","--report-id",help="ScanReport ID on the website",required=True,type=int)
 @click.option("-n","--number-of-events",help="number of rows to generate",required=True,type=int)
 @click.option("-o","--output-directory",help="folder to save the synthetic data to",required=True,type=str)
 @click.option("--fill-column-with-values",help="select columns to fill values for",multiple=True,type=str)
-@click.option("-t","--token",help="specify the carrot_token for accessing the CCOM website",type=str,default=None)
 @click.option("--get-json",help="also download the json",is_flag=True)
-@click.option("-u","--url",help="url endpoint for the CCOM website to ping",
-              type=str,
-              default="https://ccom.azurewebsites.net")
-def ccom(report_id,number_of_events,output_directory,
-         fill_column_with_values,token,get_json,
-         url):
+@click.pass_obj
+def carrot_mapper(ctx,report_id,number_of_events,output_directory,
+         fill_column_with_values,get_json):
 
-    fill_column_with_values = list(fill_column_with_values)
-
-    token = os.environ.get("carrot_TOKEN") or token
-    if token == None:
-        raise MissingToken("you must use the option --token or set the environment variable carrot_TOKEN to be able to use this functionality. I.e  export carrot_TOKEN=12345678 ")
-
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
-        "Content-type": "application/json",
-        "charset": "utf-8",
-        "Authorization": f"Token {token}"
-    }
+    url = ctx['url']
+    headers = ctx['headers']
 
+    fill_column_with_values = list(fill_column_with_values)
+
     if get_json:
         response = requests.get(
             f"{url}/api/json/?id={report_id}",
@@ -60,10 +49,9 @@ def ccom(report_id,number_of_events,output_directory,
         with open(f'{fname}.json', 'w') as outfile:
             print ('saving',fname)
             json.dump(response.json()[0],outfile,indent=6)
-
-
+
     response = requests.get(
-        f"{url}/api/scanreporttablesfilter/?scan_report={report_id}",
+        f"{url}/api/scanreporttables/?scan_report={report_id}",
         headers=headers
     )
     if response.status_code != 200:
@@ -83,7 +71,7 @@ def ccom(report_id,number_of_events,output_directory,
         if person_id == None:
             continue
 
-        _url = f"{url}/api/scanreportfieldsfilter/?id={person_id}&fields=name"
+        _url = f"{url}/api/scanreportfields/?id={person_id}&fields=name"
 
         person_id = requests.get(
             _url, headers=headers,
@@ -99,7 +87,7 @@ def ccom(report_id,number_of_events,output_directory,
 
 
         df = pd.DataFrame.from_records(response.json()).set_index('scan_report_field')        
-        _url = f"{url}/api/scanreportfieldsfilter/?scan_report_table={_id}&fields=id,name"
+        _url = f"{url}/api/scanreportfields/?scan_report_table={_id}&fields=id,name"
         response = requests.get(
             _url, headers=headers,
             allow_redirects=True,
@@ -219,8 +207,8 @@ def xlsx(report,number_of_events,output_directory,fill_column_with_values):
         print (f"created {fname} with {number_of_events} events")
 
 
-synthetic.add_command(xlsx,"from_xlsx")
-synthetic.add_command(ccom,"from_ccom")
+synthetic.add_command(xlsx,"from-xlsx")
+synthetic.add_command(carrot_mapper,"from-carrot-mapper")
 
 @click.command(help="generate a synthetic CDM from a yaml file")
 @click.argument("config")

carrot/cli/subcommands/get.py

@@ -12,30 +12,13 @@ class MissingToken(Exception):
 
 
 @click.group(help='Commands to get data from the CCOM api.')
-@click.option("-t","--token",help="specify the carrot_token for accessing the CCOM website",type=str,default=None)
+@click.option("-t","--token",help="specify the token for accessing the CCOM website",type=str,default=None)
 @click.option("-u","--url",help="url endpoint for the CCOM website to ping",
               type=str,
               default=None)
 @click.pass_context
 def get(ctx,token,url):
-    config = dotenv_values(".env")
-    if token:
-        config['CCOM_TOKEN'] = token
-    if url:
-        config['CCOM_URL'] = url
-
-    if 'CCOM_TOKEN' not in config:
-        raise MissingToken("you must use the option --token or create a .env file containing CCOM_TOKEN to be able to use this functionality.")
-
-    token = config['CCOM_TOKEN']
-    config['headers'] = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
-        "Content-type": "application/json",
-        "charset": "utf-8",
-        "Authorization": f"Token {token}"
-    }
-    ctx.obj = config
-
+    pass
 
 @click.command(help="get information about a concept")
 @click.argument("concept_id",required=True)
@@ -187,23 +170,10 @@ def concepts(config,flat):
 
 @click.command(help="get a json file")
 @click.option("-i","--report-id",help="ScanReport ID on the website",required=True,type=int)
-@click.option("-t","--token",help="specify the carrot_token for accessing the CCOM website",type=str,default=None)
-@click.option("-u","--url",help="url endpoint for the CCOM website to ping",
-              type=str,
-              default="https://ccom.azurewebsites.net")
-def _json(report_id,token,url):
-
-    token = os.environ.get("carrot_TOKEN") or token
-    if token == None:
-        raise MissingToken("you must use the option --token or set the environment variable carrot_TOKEN to be able to use this functionality. I.e  export carrot_TOKEN=12345678 ")
-
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
-        "Content-type": "application/json",
-        "charset": "utf-8",
-        "Authorization": f"Token {token}"
-    }
-
+@click.pass_obj
+def _json(ctx,report_id):
+    url = ctx['url']
+    headers = ctx['headers']
     response = requests.get(
         f"{url}/api/json/?id={report_id}",
         headers=headers

carrot/tools/__init__.py

@@ -101,3 +101,11 @@ def get_classes_from_tool(format=format):
     else:
         return retval
 
+def get_request_headers(token):
+    return {
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36",
+        "Content-type": "application/json",
+        "charset": "utf-8",
+        "Authorization": f"Token {token}"
+    }
+