From 7d3c515b35ef3928a79d26fb872fb0d1b8b40784 Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Sun, 8 Nov 2020 19:51:32 +0530 Subject: [PATCH 1/6] feat(api): add POST and PUT access method endpoints --- ...ional.data_repository_service.swagger.yaml | 92 ++++++++++++++++++- .../drs/endpoints/register_access_methods.py | 0 2 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 drs_filer/ga4gh/drs/endpoints/register_access_methods.py diff --git a/drs_filer/api/additional.data_repository_service.swagger.yaml b/drs_filer/api/additional.data_repository_service.swagger.yaml index 518a8ac..f212bfb 100644 --- a/drs_filer/api/additional.data_repository_service.swagger.yaml +++ b/drs_filer/api/additional.data_repository_service.swagger.yaml @@ -115,6 +115,50 @@ paths: - DRS-Filer x-swagger-router-controller: ga4gh.drs.server '/objects/{object_id}/access/{access_id}': + put: + summary: Create or update an access method. + description: |- + Create a access method with a predefined ID. Overwrites any + existing access method with the same ID. + operationId: PutAccessMethod + responses: + '200': + description: The `AccessMethod` was successfully created/updated. + schema: + type: string + '400': + description: The request is malformed. + schema: + $ref: '#/definitions/Error' + '401': + description: The request is unauthorized. + schema: + $ref: '#/definitions/Error' + '403': + description: The requester is not authorized to perform this action. + schema: + $ref: '#/definitions/Error' + '500': + description: An unexpected error occurred. + schema: + $ref: '#/definitions/Error' + parameters: + - name: object_id + in: path + required: true + type: string + - name: access_id + in: path + required: true + type: string + - in: body + name: AccessMethodRegister + description: Access method metadata. + schema: + $ref: '#/definitions/AccessMethodRegister' + tags: + - DRS-Filer + x-swagger-router-controller: ga4gh.drs.server delete: summary: Delete object's access method. description: >- @@ -156,7 +200,53 @@ paths: type: string tags: - DRS-Filer - x-swagger-router-controller: ga4gh.drs.server + x-swagger-router-controller: ga4gh.drs.server + + '/objects/{object_id}/access': + post: + summary: Add an object's access method. + description: >- + Add `AccessMethod` of an existing `DrsObject`. + operationId: PostAccessMethod + responses: + '200': + description: The `AccessMethod` was successfully added. + schema: + type: string + '400': + description: The request is malformed. + schema: + $ref: '#/definitions/Error' + '401': + description: The request is unauthorized. + schema: + $ref: '#/definitions/Error' + '403': + description: The requester is not authorized to perform this action. + schema: + $ref: '#/definitions/Error' + '404': + description: The requested resource was not found. + schema: + $ref: '#/definitions/Error' + '500': + description: An unexpected error occurred. + schema: + $ref: '#/definitions/Error' + parameters: + - name: object_id + in: path + required: true + type: string + - in: body + name: AccessMethodRegister + description: Access method metadata. + schema: + $ref: '#/definitions/AccessMethodRegister' + tags: + - DRS-Filer + x-swagger-router-controller: ga4gh.drs.server + definitions: DrsObjectRegister: type: object diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py new file mode 100644 index 0000000..e69de29 From 872e2131bdba0cab282f051b7ecca99d4072e7ef Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Wed, 11 Nov 2020 22:39:52 +0530 Subject: [PATCH 2/6] feat: add implementation for register_access_methods --- .../drs/endpoints/register_access_methods.py | 120 ++++++++++++++++++ drs_filer/ga4gh/drs/server.py | 20 +++ 2 files changed, 140 insertions(+) diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py index e69de29..ebfe814 100644 --- a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py +++ b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py @@ -0,0 +1,120 @@ +import logging +from random import choice +import string +from typing import Dict, Optional + +from flask import current_app + +from drs_filer.errors.exceptions import ( + ObjectNotFound, + InternalServerError +) + +logger = logging.getLogger(__name__) + + +def register_access_method( + data: Dict, + object_id: str, + access_id: Optional[str] = None, +) -> str: + """TODO: Add documentation""" + # Set parameters + db_collection = ( + current_app.config['FOCA'].db.dbs['drsStore']. + collections['objects'].client + ) + obj = db_collection.find_one({"id": object_id}) + if not obj: + raise ObjectNotFound + + # Set flags and parameters for POST/PUT routes + replace = True + if access_id is None: + replace = False + id_length = ( + current_app.config['FOCA'].endpoints['objects']['id_length'] + ) + id_charset: str = ( + current_app.config['FOCA'].endpoints['objects']['id_charset'] + ) + # evaluate character set expression or interpret literal string as set + try: + id_charset = eval(id_charset) + except Exception: + id_charset = ''.join(sorted(set(id_charset))) + + if access_id is not None: + data['access_id'] = access_id + else: + data[access_id] = generate_unique_id( + data_object=obj, + charset=id_charset, # type: ignore + length=id_length, # type: ignore + ) + + if replace: + db_collection.update_one( + filter={'id': object_id}, + update={ + '$set': { + 'access_methods.$[element]': data + } + }, + array_filters=[{'element.access_id': data['access_id']}], + upsert=True + ) + logger.info( + f"Replaced Access Method with access_id: {data['access_id']}" + " to object with id: {object_id}" + ) + else: + db_collection.update_one( + filter={'id': object_id}, + update={ + '$push': { + 'access_methods': data + } + } + ) + logger.info( + f"Added Access Method with access_id: {data['access_id']}" + " to object with id: {object_id}" + ) + return data['access_id'] + + +def generate_unique_id( + data_object: Dict, + charset: str = ''.join([string.ascii_letters, string.digits]), + length: int = 6, + retries: int = 9 +) -> str: + """Generate random string based on allowed set of characters. + + Args: + charset: String of allowed characters. + length: Length of returned string. + + Returns: + Random string of specified length and composed of defined set of + allowed characters. + """ + if('access_methods' in data_object): + access_methods = data_object['access_methods'] + access_ids = [] + for method in access_methods: + access_ids.append(method['access_id']) + for i in range(retries + 1): + logger.debug(f"Trying to generate unique id: try {i}") + access_id = ''.join(choice(charset) for __ in range(length)) + if(access_id not in access_ids): + return access_id + else: + logger.error( + f"Could not generate unique identifier." + " Tried {retries + 1} times." + ) + raise InternalServerError + else: + return ''.join(choice(charset) for __ in range(length)) diff --git a/drs_filer/ga4gh/drs/server.py b/drs_filer/ga4gh/drs/server.py index 6b916e4..c05d3a7 100644 --- a/drs_filer/ga4gh/drs/server.py +++ b/drs_filer/ga4gh/drs/server.py @@ -1,5 +1,8 @@ """Controllers for DRS endpoints.""" +from drs_filer.ga4gh.drs.endpoints.register_access_methods import ( + register_access_method +) import logging from typing import Dict @@ -165,3 +168,20 @@ def PutObject(object_id: str): data=request.json, object_id=object_id, ) + + +@log_traffic +def PostAccessMethod(object_id: str): + return register_access_method( + data=request.json, + object_id=object_id + ) + + +@log_traffic +def PutAccessMethod(object_id: str, access_id: str): + return register_access_method( + data=request.json, + object_id=object_id, + access_id=access_id + ) From e410babbcf2f881734d9c1808afae44244031dfb Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Thu, 12 Nov 2020 23:25:22 +0530 Subject: [PATCH 3/6] docs and refactoring --- .../drs/endpoints/register_access_methods.py | 59 ++++++++++++------- drs_filer/ga4gh/drs/server.py | 20 +++++++ 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py index ebfe814..ef11768 100644 --- a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py +++ b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py @@ -11,14 +11,22 @@ ) logger = logging.getLogger(__name__) - - + def register_access_method( data: Dict, object_id: str, access_id: Optional[str] = None, ) -> str: - """TODO: Add documentation""" + """Register access method. + + Args: + data: Request object of type `AccessMethodRegister`. + object_id: DRS object identifier. + access_id: Access method identifier. Auto-generated if not provided. + + Returns: + A unique identifier for the access method. + """ # Set parameters db_collection = ( current_app.config['FOCA'].db.dbs['drsStore']. @@ -47,14 +55,14 @@ def register_access_method( if access_id is not None: data['access_id'] = access_id else: - data[access_id] = generate_unique_id( + data['access_id'] = generate_unique_id( data_object=obj, charset=id_charset, # type: ignore length=id_length, # type: ignore ) if replace: - db_collection.update_one( + result = db_collection.update_one( filter={'id': object_id}, update={ '$set': { @@ -62,25 +70,33 @@ def register_access_method( } }, array_filters=[{'element.access_id': data['access_id']}], - upsert=True - ) - logger.info( - f"Replaced Access Method with access_id: {data['access_id']}" - " to object with id: {object_id}" ) - else: - db_collection.update_one( - filter={'id': object_id}, - update={ - '$push': { - 'access_methods': data - } + + if(result.modified_count): + logger.info( + f"Replaced access method with access_id: {data['access_id']}" + f" of DRS object with id: {object_id}" + ) + return data['access_id'] + + # Try adding the access method incase of POST or incase + # no element matches with the filter in case of replacement. + result = db_collection.update_one( + filter={'id': object_id}, + update={ + '$push': { + 'access_methods': data } - ) + } + ) + if(result.modified_count): logger.info( - f"Added Access Method with access_id: {data['access_id']}" - " to object with id: {object_id}" + f"Added access method with access_id: {data['access_id']}" + f" to DRS object with id: {object_id}" ) + # Access method neither added nor updated. + else: + raise InternalServerError return data['access_id'] @@ -93,8 +109,11 @@ def generate_unique_id( """Generate random string based on allowed set of characters. Args: + data_object: The DRS object charset: String of allowed characters. length: Length of returned string. + retries: If `access_id` is not supplied, how many times should the + generation of a random identifier take place. Returns: Random string of specified length and composed of defined set of diff --git a/drs_filer/ga4gh/drs/server.py b/drs_filer/ga4gh/drs/server.py index c05d3a7..f7832a2 100644 --- a/drs_filer/ga4gh/drs/server.py +++ b/drs_filer/ga4gh/drs/server.py @@ -172,6 +172,15 @@ def PutObject(object_id: str): @log_traffic def PostAccessMethod(object_id: str): + """Add a new `AccessMethod` in an existing DRS object + + Args: + object_id: Identifier of the DRS object, in which the + `AccessMethod` is to be added. + + Returns: + Identifier of the added `AccessMethod`. + """ return register_access_method( data=request.json, object_id=object_id @@ -180,6 +189,17 @@ def PostAccessMethod(object_id: str): @log_traffic def PutAccessMethod(object_id: str, access_id: str): + """Add/replace an `AccessMethod` in an existing DRS object + + Args: + object_id: Identifier of the DRS object, to which the + `AccessMethod` is to be added. + access_id: Identifier of the `AccessMethod` which is to + be added/updated. + + Returns: + Identifier of the added/updated `AccessMethod`. + """ return register_access_method( data=request.json, object_id=object_id, From bcd32b660c8c77732d5eb45acc5dd438ee01dcc4 Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Thu, 12 Nov 2020 23:29:05 +0530 Subject: [PATCH 4/6] refactor: flake8 fix --- drs_filer/ga4gh/drs/endpoints/register_access_methods.py | 5 +++-- drs_filer/ga4gh/drs/server.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py index ef11768..b169f72 100644 --- a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py +++ b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py @@ -11,7 +11,8 @@ ) logger = logging.getLogger(__name__) - + + def register_access_method( data: Dict, object_id: str, @@ -78,7 +79,7 @@ def register_access_method( f" of DRS object with id: {object_id}" ) return data['access_id'] - + # Try adding the access method incase of POST or incase # no element matches with the filter in case of replacement. result = db_collection.update_one( diff --git a/drs_filer/ga4gh/drs/server.py b/drs_filer/ga4gh/drs/server.py index f7832a2..26735b3 100644 --- a/drs_filer/ga4gh/drs/server.py +++ b/drs_filer/ga4gh/drs/server.py @@ -194,7 +194,7 @@ def PutAccessMethod(object_id: str, access_id: str): Args: object_id: Identifier of the DRS object, to which the `AccessMethod` is to be added. - access_id: Identifier of the `AccessMethod` which is to + access_id: Identifier of the `AccessMethod` which is to be added/updated. Returns: From 08c320450d76c6acb30538afaf0e6ec33e528490 Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Sat, 21 Nov 2020 21:33:14 +0530 Subject: [PATCH 5/6] refactor: new implementation --- .../drs/endpoints/register_access_methods.py | 114 ++++++++---------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py index b169f72..4dc274e 100644 --- a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py +++ b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py @@ -17,6 +17,7 @@ def register_access_method( data: Dict, object_id: str, access_id: Optional[str] = None, + retries: int = 9 ) -> str: """Register access method. @@ -24,6 +25,9 @@ def register_access_method( data: Request object of type `AccessMethodRegister`. object_id: DRS object identifier. access_id: Access method identifier. Auto-generated if not provided. + retries: If `access_id` is not supplied, how many times should the + generation of a random identifier and insertion into the database + be retried in case of duplicate access_ids. Returns: A unique identifier for the access method. @@ -53,88 +57,76 @@ def register_access_method( except Exception: id_charset = ''.join(sorted(set(id_charset))) - if access_id is not None: - data['access_id'] = access_id - else: - data['access_id'] = generate_unique_id( - data_object=obj, - charset=id_charset, # type: ignore - length=id_length, # type: ignore - ) + # Try to generate unique ID and insert object into database + for i in range(retries + 1): + logger.debug(f"Trying to insert/update access method: try {i}") + # Set or generate object identifier + if access_id is not None: + data['access_id'] = access_id + else: + data['access_id'] = generate_id( + charset=id_charset, # type: ignore + length=id_length, # type: ignore + ) + # Replace access method, then return (PUT) + if replace: + result_replace = db_collection.update_one( + filter={'id': object_id}, + update={ + '$set': { + 'access_methods.$[element]': data + } + }, + array_filters=[{'element.access_id': data['access_id']}], + ) - if replace: - result = db_collection.update_one( - filter={'id': object_id}, + if(result_replace.modified_count): + logger.info( + f"Replaced access method with access_id: " + f"{data['access_id']} of DRS object with id: {object_id}" + ) + break + + # Try inserting the access method incase of POST or incase + # no element matches with the filter incase of PUT. + result_insert = db_collection.update_one( + filter={ + 'id': object_id, + 'access_methods.access_id': {'$ne': data['access_id']} + }, update={ - '$set': { - 'access_methods.$[element]': data + '$push': { + 'access_methods': data } - }, - array_filters=[{'element.access_id': data['access_id']}], + } ) - - if(result.modified_count): + if(result_insert.modified_count): logger.info( - f"Replaced access method with access_id: {data['access_id']}" - f" of DRS object with id: {object_id}" + f"Added access method with access_id: {data['access_id']}" + f" to DRS object with id: {object_id}" ) - return data['access_id'] - - # Try adding the access method incase of POST or incase - # no element matches with the filter in case of replacement. - result = db_collection.update_one( - filter={'id': object_id}, - update={ - '$push': { - 'access_methods': data - } - } - ) - if(result.modified_count): - logger.info( - f"Added access method with access_id: {data['access_id']}" - f" to DRS object with id: {object_id}" - ) + break # Access method neither added nor updated. else: + logger.error( + f"Could not generate unique identifier. Tried {retries + 1} times." + ) raise InternalServerError return data['access_id'] -def generate_unique_id( - data_object: Dict, +def generate_id( charset: str = ''.join([string.ascii_letters, string.digits]), - length: int = 6, - retries: int = 9 + length: int = 6 ) -> str: """Generate random string based on allowed set of characters. Args: - data_object: The DRS object charset: String of allowed characters. length: Length of returned string. - retries: If `access_id` is not supplied, how many times should the - generation of a random identifier take place. Returns: Random string of specified length and composed of defined set of allowed characters. """ - if('access_methods' in data_object): - access_methods = data_object['access_methods'] - access_ids = [] - for method in access_methods: - access_ids.append(method['access_id']) - for i in range(retries + 1): - logger.debug(f"Trying to generate unique id: try {i}") - access_id = ''.join(choice(charset) for __ in range(length)) - if(access_id not in access_ids): - return access_id - else: - logger.error( - f"Could not generate unique identifier." - " Tried {retries + 1} times." - ) - raise InternalServerError - else: - return ''.join(choice(charset) for __ in range(length)) + return ''.join(choice(charset) for __ in range(length)) From 1787452049685cc4c204e0b039635c3c7bf20b63 Mon Sep 17 00:00:00 2001 From: Sarthak Gupta Date: Sat, 21 Nov 2020 21:34:50 +0530 Subject: [PATCH 6/6] minor doc change --- drs_filer/ga4gh/drs/endpoints/register_access_methods.py | 1 + 1 file changed, 1 insertion(+) diff --git a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py index 4dc274e..d2ceee8 100644 --- a/drs_filer/ga4gh/drs/endpoints/register_access_methods.py +++ b/drs_filer/ga4gh/drs/endpoints/register_access_methods.py @@ -39,6 +39,7 @@ def register_access_method( ) obj = db_collection.find_one({"id": object_id}) if not obj: + logger.error(f"DRS object with id: {object_id} not found.") raise ObjectNotFound # Set flags and parameters for POST/PUT routes