diff --git a/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java index 41802f93..f92d842e 100644 --- a/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java +++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/TransferOrchestrator.java @@ -106,7 +106,7 @@ public void processTransfer(String transferId, AgentTransferRequest request, MetadataCollector srcMetadataCollector = srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a metadata collector for source")); srcMetadataCollector.init(request.getSourceStorage(), request.getSourceSecret()); - ResourceMetadata srcMetadata = srcMetadataCollector.getResourceMetadata(request.getSourcePath()); + ResourceMetadata srcMetadata = srcMetadataCollector.getResourceMetadata(request.getSourcePath(), false); if (srcMetadata.getMetadataCase() != ResourceMetadata.MetadataCase.FILE) { throw new Exception("Expected a file as the source but received " + srcMetadata.getMetadataCase().name()); } diff --git a/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java b/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java index 1dcc6709..bb8d81de 100644 --- a/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java +++ b/agent/service/src/main/java/org/apache/airavata/mft/agent/rpc/RPCParser.java @@ -50,7 +50,7 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception { if (metadataCollectorOptional.isPresent()) { MetadataCollector metadataCollector = metadataCollectorOptional.get(); metadataCollector.init(req.getStorage(), req.getSecret()); - ResourceMetadata resourceMetadata = metadataCollector.getResourceMetadata(req.getResourcePath()); + ResourceMetadata resourceMetadata = metadataCollector.getResourceMetadata(req.getResourcePath(), directResourceMetadataReq.getRecursiveSearch()); return JsonFormat.printer().print(resourceMetadata); } else { throw new Exception("No metadata collector for type " + req.getStorage().getStorageCase().name()); diff --git a/agent/stub/src/main/proto/MFTAgentStubs.proto b/agent/stub/src/main/proto/MFTAgentStubs.proto index 17eda3b8..24041a10 100644 --- a/agent/stub/src/main/proto/MFTAgentStubs.proto +++ b/agent/stub/src/main/proto/MFTAgentStubs.proto @@ -114,5 +114,6 @@ message GetResourceMetadataRequest { string resourcePath = 1; StorageWrapper storage = 2; SecretWrapper secret = 3; + bool recursiveSearch = 4; } diff --git a/api/service/pom.xml b/api/service/pom.xml index fdf6f6fb..7f316c5b 100644 --- a/api/service/pom.xml +++ b/api/service/pom.xml @@ -78,13 +78,15 @@ - - org.apache.airavata - mft-scp-transport - 0.01-SNAPSHOT - runtime + mft-resource-service-client + ${project.version} + + + org.apache.airavata + mft-secret-service-client + ${project.version} diff --git a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java index 60a856e0..1ae2139c 100644 --- a/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java +++ b/api/service/src/main/java/org/apache/airavata/mft/api/handler/MFTApiHandler.java @@ -28,6 +28,50 @@ import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse; import org.apache.airavata.mft.agent.stub.*; import org.apache.airavata.mft.api.service.*; +import org.apache.airavata.mft.credential.stubs.azure.AzureSecret; +import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.box.BoxSecret; +import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret; +import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret; +import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret; +import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecret; +import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.s3.S3Secret; +import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest; +import org.apache.airavata.mft.credential.stubs.scp.SCPSecret; +import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret; +import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest; +import org.apache.airavata.mft.resource.client.StorageServiceClient; +import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder; +import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage; +import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage; +import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorage; +import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage; +import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage; +import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorage; +import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage; +import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage; +import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest; +import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorage; +import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageGetRequest; +import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest; +import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage; +import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest; +import org.apache.airavata.mft.secret.client.SecretServiceClient; +import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder; import org.apache.commons.lang3.tuple.Pair; import org.dozer.DozerBeanMapper; import org.lognet.springboot.grpc.GRpcService; @@ -243,10 +287,151 @@ public void getTransferState(TransferStateApiRequest request, StreamObserver responseObserver) { - GetResourceMetadataRequest directRequest = request.getDirectRequest(); + GetResourceMetadataRequest directRequest = null; + try { + if (request.getRequestCase() == FetchResourceMetadataRequest.RequestCase.DIRECTREQUEST) { + directRequest = request.getDirectRequest(); + } else { + directRequest = deriveDirectRequest(request.getIdRequest()); + } + String targetAgent = derriveTargetAgent(""); SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder() .withAgentId(targetAgent) @@ -282,8 +467,16 @@ public void getResourceAvailability(FetchResourceMetadataRequest request, Stream @Override public void resourceMetadata(FetchResourceMetadataRequest request, StreamObserver responseObserver) { - GetResourceMetadataRequest directRequest = request.getDirectRequest(); + GetResourceMetadataRequest directRequest = null; + try { + + if (request.getRequestCase() == FetchResourceMetadataRequest.RequestCase.DIRECTREQUEST) { + directRequest = request.getDirectRequest(); + } else { + directRequest = deriveDirectRequest(request.getIdRequest()); + } + String targetAgent = derriveTargetAgent(""); SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder() .withAgentId(targetAgent) diff --git a/api/stub/src/main/proto/MFTTransferApi.proto b/api/stub/src/main/proto/MFTTransferApi.proto index 4cb5a533..ca214599 100644 --- a/api/stub/src/main/proto/MFTTransferApi.proto +++ b/api/stub/src/main/proto/MFTTransferApi.proto @@ -86,6 +86,7 @@ message GetResourceMetadataFromIDsRequest { string resourcePath = 1; string storageId = 2; string secretId = 3; + bool recursiveSearch = 4; } message FetchResourceMetadataRequest { diff --git a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java index 92811fa2..4f265b59 100644 --- a/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java +++ b/core/src/main/java/org/apache/airavata/mft/core/api/MetadataCollector.java @@ -20,7 +20,6 @@ import org.apache.airavata.mft.agent.stub.ResourceMetadata; import org.apache.airavata.mft.agent.stub.SecretWrapper; import org.apache.airavata.mft.agent.stub.StorageWrapper; -import org.apache.airavata.mft.core.DirectoryResourceMetadata; import org.apache.airavata.mft.core.FileResourceMetadata; public interface MetadataCollector { @@ -36,11 +35,12 @@ public interface MetadataCollector { /** * Fetches a metadata of given File Resource * - * @param resourcePath path of the resource + * @param resourcePath path of the resource + * @param recursiveSearch * @return an object of {@link FileResourceMetadata} * @throws Exception if the resource id is not a File Resource type or the resource can't be fetched from the resource service */ - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception; + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception; /** * Check whether the resource is available in the actual storage diff --git a/python-cli/README.md b/python-cli/README.md index 200641d6..2474765d 100644 --- a/python-cli/README.md +++ b/python-cli/README.md @@ -18,7 +18,7 @@ Install dependencies ``` pip install grpcio==1.46.3 pip install grpcio-tools==1.46.3 -pip install airavata_mft_sdk==0.0.1-alpha15 +pip install airavata_mft_sdk==0.0.1-alpha18 ``` Build the binary diff --git a/python-cli/mft_cli/mft_cli/main.py b/python-cli/mft_cli/mft_cli/main.py index 9dc65344..db1109d4 100644 --- a/python-cli/mft_cli/mft_cli/main.py +++ b/python-cli/mft_cli/mft_cli/main.py @@ -1,11 +1,74 @@ import typer import mft_cli.storage +from airavata_mft_sdk import mft_client +from airavata_mft_sdk.common import StorageCommon_pb2 +from airavata_mft_sdk import MFTTransferApi_pb2 +from rich.console import Console +from rich.table import Table app = typer.Typer() -storage_app = typer.Typer() app.add_typer(mft_cli.storage.app, name="storage") +@app.command("ls") +def list(storage_path): + storage_name = storage_path.split("/")[0] + resource_path = storage_path[len(storage_name) +1 :] + client = mft_client.MFTClient() + search_req = StorageCommon_pb2.StorageSearchRequest(storageName=storage_name) + storages = client.common_api.searchStorages(search_req) + + if len(storages.storageList) == 0: + search_req = StorageCommon_pb2.StorageSearchRequest(storageId=storage_name) + storages = client.common_api.searchStorages(search_req) + + if len(storages.storageList) == 0: + print("No storage with name or id " + storage_name + " was found. Please register the storage with command mft-cli storage add") + exit() + + if len(storages.storageList) > 1: + print("More than one storage with nam " + storage_name + " was found. Please use the storage id. You can fetch it from mft-cli storage list") + exit() + + storage = storages.storageList[0] + sec_req = StorageCommon_pb2.SecretForStorageGetRequest(storageId = storage.storageId) + sec_resp = client.common_api.getSecretForStorage(sec_req) + if sec_resp.error != 0: + print("Could not fetch the secret for storage " + storage.storageId) + + id_req = MFTTransferApi_pb2.GetResourceMetadataFromIDsRequest(storageId = sec_resp.storageId, + secretId = sec_resp.secretId, + resourcePath = resource_path) + resource_medata_req = MFTTransferApi_pb2.FetchResourceMetadataRequest(idRequest = id_req) + + metadata_resp = client.transfer_api.resourceMetadata(resource_medata_req) + + console = Console() + table = Table() + + table.add_column('Name', justify='left') + table.add_column('Type', justify='center') + table.add_column('Size', justify='center') + + if (metadata_resp.WhichOneof('metadata') == 'directory') : + for dir in metadata_resp.directory.directories: + table.add_row('[bold]' + dir.friendlyName + '[/bold]', 'DIR', '') + + for file in metadata_resp.directory.files: + table.add_row('[bold]' + file.friendlyName + '[/bold]', 'FILE', str(file.resourceSize)) + + elif (metadata_resp.WhichOneof('metadata') == 'file'): + table.add_row('[bold]' + metadata_resp.file.friendlyName + '[/bold]', 'FILE', str(metadata_resp.file.resourceSize)) + + elif (metadata_resp.WhichOneof('metadata') == 'error'): + print(metadata_resp.error) + + console.print(table) + +@app.command("cp") +def copy(source, destination): + print("Moving data from " + source + " to " + destination) + if __name__ == "__main__": app() \ No newline at end of file diff --git a/python-cli/mft_cli/mft_cli/storage/s3.py b/python-cli/mft_cli/mft_cli/storage/s3.py index cfe1b1b9..5b8881e5 100644 --- a/python-cli/mft_cli/mft_cli/storage/s3.py +++ b/python-cli/mft_cli/mft_cli/storage/s3.py @@ -86,10 +86,10 @@ def handle_add_storage(): selected_bucket = typer.prompt("Enter bucket name ") storage_name = typer.prompt("Name of the storage ", selected_bucket) - s3_storage.bucketName = selected_bucket - s3_storage.name = storage_name + s3_storage_create_req = S3Storage_pb2.S3StorageCreateRequest( + endpoint=endpoint, region=region, bucketName=selected_bucket, name =storage_name) - created_storage = client.s3_storage_api.createS3Storage(s3_storage) + created_storage = client.s3_storage_api.createS3Storage(s3_storage_create_req) secret_create_req= S3Credential_pb2.S3SecretCreateRequest(accessKey=client_id, secretKey=client_secret) created_secret = client.s3_secret_api.createS3Secret(secret_create_req) diff --git a/python-sdk/setup.cfg b/python-sdk/setup.cfg index 66ecc6cf..0a18af5c 100644 --- a/python-sdk/setup.cfg +++ b/python-sdk/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = airavata_mft_sdk -version = 0.0.1-alpha16 +version = 0.0.1-alpha18 author = Airavata MFT Developers author_email = dev@airavata.apache.org description = Python SDK for Apache Airavata Managed File Transfers (MFT) diff --git a/python-sdk/src/airavata_mft_sdk/MFTAgentStubs_pb2.py b/python-sdk/src/airavata_mft_sdk/MFTAgentStubs_pb2.py index a72c134b..b1192d6b 100644 --- a/python-sdk/src/airavata_mft_sdk/MFTAgentStubs_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/MFTAgentStubs_pb2.py @@ -34,7 +34,7 @@ from airavata_mft_sdk.swift import SwiftCredential_pb2 as swift_dot_SwiftCredential__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13MFTAgentStubs.proto\x12\"org.apache.airavata.mft.agent.stub\x1a\x18\x61zure/AzureStorage.proto\x1a\x14\x62ox/BoxStorage.proto\x1a\x1c\x64ropbox/DropboxStorage.proto\x1a\x14\x66tp/FTPStorage.proto\x1a\x14gcs/GCSStorage.proto\x1a\x18local/LocalStorage.proto\x1a\x18odata/ODataStorage.proto\x1a\x12s3/S3Storage.proto\x1a\x14scp/SCPStorage.proto\x1a\x18swift/SwiftStorage.proto\x1a\x1b\x61zure/AzureCredential.proto\x1a\x17\x62ox/BoxCredential.proto\x1a\x1f\x64ropbox/DropboxCredential.proto\x1a\x17\x66tp/FTPCredential.proto\x1a\x17gcs/GCSCredential.proto\x1a\x1bodata/ODataCredential.proto\x1a\x15s3/S3Credential.proto\x1a\x17scp/SCPCredential.proto\x1a\x1bswift/SwiftCredential.proto\"\xd2\x06\n\x0eStorageWrapper\x12S\n\x05\x61zure\x18\x01 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageH\x00\x12M\n\x03\x62ox\x18\x02 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.box.storage.BoxStorageH\x00\x12Y\n\x07\x64ropbox\x18\x03 \x01(\x0b\x32\x46.org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorageH\x00\x12M\n\x03\x66tp\x18\x04 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageH\x00\x12M\n\x03gcs\x18\x05 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageH\x00\x12S\n\x05local\x18\x06 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.local.storage.LocalStorageH\x00\x12S\n\x05odata\x18\x07 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageH\x00\x12J\n\x02s3\x18\x08 \x01(\x0b\x32<.org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageH\x00\x12M\n\x03scp\x18\t \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageH\x00\x12S\n\x05swift\x18\n \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageH\x00\x42\t\n\x07storage\"\xbc\x05\n\rSecretWrapper\x12L\n\x05\x61zure\x18\x01 \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.azure.AzureSecretH\x00\x12\x46\n\x03\x62ox\x18\x02 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.box.BoxSecretH\x00\x12R\n\x07\x64ropbox\x18\x03 \x01(\x0b\x32?.org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretH\x00\x12\x46\n\x03\x66tp\x18\x04 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.ftp.FTPSecretH\x00\x12\x46\n\x03gcs\x18\x05 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.gcs.GCSSecretH\x00\x12L\n\x05odata\x18\x06 \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.odata.ODataSecretH\x00\x12\x43\n\x02s3\x18\x07 \x01(\x0b\x32\x35.org.apache.airavata.mft.credential.stubs.s3.S3SecretH\x00\x12\x46\n\x03scp\x18\x08 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.scp.SCPSecretH\x00\x12L\n\x05swift\x18\t \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.swift.SwiftSecretH\x00\x42\x08\n\x06secret\"\xf5\x02\n\x14\x41gentTransferRequest\x12I\n\rsourceStorage\x18\x01 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12G\n\x0csourceSecret\x18\x02 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper\x12\x12\n\nsourcePath\x18\x03 \x01(\t\x12N\n\x12\x64\x65stinationStorage\x18\x04 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12L\n\x11\x64\x65stinationSecret\x18\x05 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper\x12\x17\n\x0f\x64\x65stinationPath\x18\x06 \x01(\t\"\x89\x01\n\x0c\x46ileMetadata\x12\x14\n\x0c\x66riendlyName\x18\x01 \x01(\t\x12\x14\n\x0cresourceSize\x18\x02 \x01(\x03\x12\x13\n\x0b\x63reatedTime\x18\x03 \x01(\x03\x12\x12\n\nupdateTime\x18\x04 \x01(\x03\x12\x0e\n\x06md5sum\x18\x05 \x01(\t\x12\x14\n\x0cresourcePath\x18\x06 \x01(\t\"\xf5\x01\n\x11\x44irectoryMetadata\x12\x14\n\x0c\x66riendlyName\x18\x01 \x01(\t\x12\x13\n\x0b\x63reatedTime\x18\x02 \x01(\x03\x12\x12\n\nupdateTime\x18\x03 \x01(\x03\x12\x14\n\x0cresourcePath\x18\x04 \x01(\t\x12J\n\x0b\x64irectories\x18\x05 \x03(\x0b\x32\x35.org.apache.airavata.mft.agent.stub.DirectoryMetadata\x12?\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x30.org.apache.airavata.mft.agent.stub.FileMetadata\"\xf5\x01\n\x10ResourceMetadata\x12@\n\x04\x66ile\x18\x01 \x01(\x0b\x32\x30.org.apache.airavata.mft.agent.stub.FileMetadataH\x00\x12J\n\tdirectory\x18\x02 \x01(\x0b\x32\x35.org.apache.airavata.mft.agent.stub.DirectoryMetadataH\x00\x12G\n\x05\x65rror\x18\x03 \x01(\x0e\x32\x36.org.apache.airavata.mft.agent.stub.MetadataFetchErrorH\x00\x42\n\n\x08metadata\"\xba\x01\n\x1aGetResourceMetadataRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x43\n\x07storage\x18\x02 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12\x41\n\x06secret\x18\x03 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper*6\n\x12MetadataFetchError\x12\r\n\tNOT_FOUND\x10\x00\x12\x11\n\rNO_PERMISSION\x10\x01\x42\x02P\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13MFTAgentStubs.proto\x12\"org.apache.airavata.mft.agent.stub\x1a\x18\x61zure/AzureStorage.proto\x1a\x14\x62ox/BoxStorage.proto\x1a\x1c\x64ropbox/DropboxStorage.proto\x1a\x14\x66tp/FTPStorage.proto\x1a\x14gcs/GCSStorage.proto\x1a\x18local/LocalStorage.proto\x1a\x18odata/ODataStorage.proto\x1a\x12s3/S3Storage.proto\x1a\x14scp/SCPStorage.proto\x1a\x18swift/SwiftStorage.proto\x1a\x1b\x61zure/AzureCredential.proto\x1a\x17\x62ox/BoxCredential.proto\x1a\x1f\x64ropbox/DropboxCredential.proto\x1a\x17\x66tp/FTPCredential.proto\x1a\x17gcs/GCSCredential.proto\x1a\x1bodata/ODataCredential.proto\x1a\x15s3/S3Credential.proto\x1a\x17scp/SCPCredential.proto\x1a\x1bswift/SwiftCredential.proto\"\xd2\x06\n\x0eStorageWrapper\x12S\n\x05\x61zure\x18\x01 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageH\x00\x12M\n\x03\x62ox\x18\x02 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.box.storage.BoxStorageH\x00\x12Y\n\x07\x64ropbox\x18\x03 \x01(\x0b\x32\x46.org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorageH\x00\x12M\n\x03\x66tp\x18\x04 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageH\x00\x12M\n\x03gcs\x18\x05 \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageH\x00\x12S\n\x05local\x18\x06 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.local.storage.LocalStorageH\x00\x12S\n\x05odata\x18\x07 \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageH\x00\x12J\n\x02s3\x18\x08 \x01(\x0b\x32<.org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageH\x00\x12M\n\x03scp\x18\t \x01(\x0b\x32>.org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageH\x00\x12S\n\x05swift\x18\n \x01(\x0b\x32\x42.org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageH\x00\x42\t\n\x07storage\"\xbc\x05\n\rSecretWrapper\x12L\n\x05\x61zure\x18\x01 \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.azure.AzureSecretH\x00\x12\x46\n\x03\x62ox\x18\x02 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.box.BoxSecretH\x00\x12R\n\x07\x64ropbox\x18\x03 \x01(\x0b\x32?.org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretH\x00\x12\x46\n\x03\x66tp\x18\x04 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.ftp.FTPSecretH\x00\x12\x46\n\x03gcs\x18\x05 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.gcs.GCSSecretH\x00\x12L\n\x05odata\x18\x06 \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.odata.ODataSecretH\x00\x12\x43\n\x02s3\x18\x07 \x01(\x0b\x32\x35.org.apache.airavata.mft.credential.stubs.s3.S3SecretH\x00\x12\x46\n\x03scp\x18\x08 \x01(\x0b\x32\x37.org.apache.airavata.mft.credential.stubs.scp.SCPSecretH\x00\x12L\n\x05swift\x18\t \x01(\x0b\x32;.org.apache.airavata.mft.credential.stubs.swift.SwiftSecretH\x00\x42\x08\n\x06secret\"\xf5\x02\n\x14\x41gentTransferRequest\x12I\n\rsourceStorage\x18\x01 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12G\n\x0csourceSecret\x18\x02 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper\x12\x12\n\nsourcePath\x18\x03 \x01(\t\x12N\n\x12\x64\x65stinationStorage\x18\x04 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12L\n\x11\x64\x65stinationSecret\x18\x05 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper\x12\x17\n\x0f\x64\x65stinationPath\x18\x06 \x01(\t\"\x89\x01\n\x0c\x46ileMetadata\x12\x14\n\x0c\x66riendlyName\x18\x01 \x01(\t\x12\x14\n\x0cresourceSize\x18\x02 \x01(\x03\x12\x13\n\x0b\x63reatedTime\x18\x03 \x01(\x03\x12\x12\n\nupdateTime\x18\x04 \x01(\x03\x12\x0e\n\x06md5sum\x18\x05 \x01(\t\x12\x14\n\x0cresourcePath\x18\x06 \x01(\t\"\xf5\x01\n\x11\x44irectoryMetadata\x12\x14\n\x0c\x66riendlyName\x18\x01 \x01(\t\x12\x13\n\x0b\x63reatedTime\x18\x02 \x01(\x03\x12\x12\n\nupdateTime\x18\x03 \x01(\x03\x12\x14\n\x0cresourcePath\x18\x04 \x01(\t\x12J\n\x0b\x64irectories\x18\x05 \x03(\x0b\x32\x35.org.apache.airavata.mft.agent.stub.DirectoryMetadata\x12?\n\x05\x66iles\x18\x06 \x03(\x0b\x32\x30.org.apache.airavata.mft.agent.stub.FileMetadata\"\xf5\x01\n\x10ResourceMetadata\x12@\n\x04\x66ile\x18\x01 \x01(\x0b\x32\x30.org.apache.airavata.mft.agent.stub.FileMetadataH\x00\x12J\n\tdirectory\x18\x02 \x01(\x0b\x32\x35.org.apache.airavata.mft.agent.stub.DirectoryMetadataH\x00\x12G\n\x05\x65rror\x18\x03 \x01(\x0e\x32\x36.org.apache.airavata.mft.agent.stub.MetadataFetchErrorH\x00\x42\n\n\x08metadata\"\xd3\x01\n\x1aGetResourceMetadataRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x43\n\x07storage\x18\x02 \x01(\x0b\x32\x32.org.apache.airavata.mft.agent.stub.StorageWrapper\x12\x41\n\x06secret\x18\x03 \x01(\x0b\x32\x31.org.apache.airavata.mft.agent.stub.SecretWrapper\x12\x17\n\x0frecursiveSearch\x18\x04 \x01(\x08*6\n\x12MetadataFetchError\x12\r\n\tNOT_FOUND\x10\x00\x12\x11\n\rNO_PERMISSION\x10\x01\x42\x02P\x01\x62\x06proto3') _METADATAFETCHERROR = DESCRIPTOR.enum_types_by_name['MetadataFetchError'] MetadataFetchError = enum_type_wrapper.EnumTypeWrapper(_METADATAFETCHERROR) @@ -102,8 +102,8 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'P\001' - _METADATAFETCHERROR._serialized_start=3301 - _METADATAFETCHERROR._serialized_end=3355 + _METADATAFETCHERROR._serialized_start=3326 + _METADATAFETCHERROR._serialized_end=3380 _STORAGEWRAPPER._serialized_start=545 _STORAGEWRAPPER._serialized_end=1395 _SECRETWRAPPER._serialized_start=1398 @@ -117,5 +117,5 @@ _RESOURCEMETADATA._serialized_start=2865 _RESOURCEMETADATA._serialized_end=3110 _GETRESOURCEMETADATAREQUEST._serialized_start=3113 - _GETRESOURCEMETADATAREQUEST._serialized_end=3299 + _GETRESOURCEMETADATAREQUEST._serialized_end=3324 # @@protoc_insertion_point(module_scope) diff --git a/python-sdk/src/airavata_mft_sdk/MFTTransferApi_pb2.py b/python-sdk/src/airavata_mft_sdk/MFTTransferApi_pb2.py index 75796268..ccfb3ff5 100644 --- a/python-sdk/src/airavata_mft_sdk/MFTTransferApi_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/MFTTransferApi_pb2.py @@ -16,7 +16,7 @@ import airavata_mft_sdk.MFTAgentStubs_pb2 as MFTAgentStubs__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14MFTTransferApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x10\x43redCommon.proto\x1a\x13MFTAgentStubs.proto\"\x9b\x01\n\x10\x43\x61llbackEndpoint\x12P\n\x04type\x18\x01 \x01(\x0e\x32\x42.org.apache.airavata.mft.api.service.CallbackEndpoint.CallbackType\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\"#\n\x0c\x43\x61llbackType\x12\x08\n\x04HTTP\x10\x00\x12\t\n\x05KAFKA\x10\x01\"\xf3\x03\n\x12TransferApiRequest\x12\x12\n\nsourcePath\x18\x01 \x01(\t\x12\x17\n\x0fsourceStorageId\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x17\n\x0f\x64\x65stinationPath\x18\x04 \x01(\t\x12\x1c\n\x14\x64\x65stinationStorageId\x18\x05 \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\x06 \x01(\t\x12\x18\n\x10\x61\x66\x66inityTransfer\x18\x07 \x01(\x08\x12_\n\x0ctargetAgents\x18\x08 \x03(\x0b\x32I.org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry\x12H\n\x15mftAuthorizationToken\x18\t \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\x12P\n\x11\x63\x61llbackEndpoints\x18\n \x03(\x0b\x32\x35.org.apache.airavata.mft.api.service.CallbackEndpoint\x1a\x33\n\x11TargetAgentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\")\n\x13TransferApiResponse\x12\x12\n\ntransferId\x18\x01 \x01(\t\"l\n\x17\x42\x61tchTransferApiRequest\x12Q\n\x10transferRequests\x18\x01 \x03(\x0b\x32\x37.org.apache.airavata.mft.api.service.TransferApiRequest\"/\n\x18\x42\x61tchTransferApiResponse\x12\x13\n\x0btransferIds\x18\x01 \x03(\t\"\xc3\x01\n\x14HttpUploadApiRequest\x12\x1c\n\x14\x64\x65stinationStorageId\x18\x01 \x01(\t\x12\x14\n\x0cresourcePath\x18\x02 \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\x03 \x01(\t\x12\x13\n\x0btargetAgent\x18\x05 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x06 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\"9\n\x15HttpUploadApiResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0btargetAgent\x18\x02 \x01(\t\"\xbb\x01\n\x16HttpDownloadApiRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x17\n\x0fsourceStorageId\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x13\n\x0btargetAgent\x18\x05 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x06 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\";\n\x17HttpDownloadApiResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0btargetAgent\x18\x02 \x01(\t\"w\n\x17TransferStateApiRequest\x12\x12\n\ntransferId\x18\x01 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x02 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\"j\n\x18TransferStateApiResponse\x12\r\n\x05state\x18\x01 \x01(\t\x12\x16\n\x0eupdateTimeMils\x18\x02 \x01(\x03\x12\x12\n\npercentage\x18\x03 \x01(\x01\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\"1\n\x1cResourceAvailabilityResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\"^\n!GetResourceMetadataFromIDsRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x11\n\tstorageId\x18\x02 \x01(\t\x12\x10\n\x08secretId\x18\x03 \x01(\t\"\xa9\x02\n\x1c\x46\x65tchResourceMetadataRequest\x12W\n\rdirectRequest\x18\x01 \x01(\x0b\x32>.org.apache.airavata.mft.agent.stub.GetResourceMetadataRequestH\x00\x12[\n\tidRequest\x18\x02 \x01(\x0b\x32\x46.org.apache.airavata.mft.api.service.GetResourceMetadataFromIDsRequestH\x00\x12H\n\x15mftAuthorizationToken\x18\x03 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthTokenB\t\n\x07request2\xa4\t\n\x12MFTTransferService\x12\x83\x01\n\x0esubmitTransfer\x12\x37.org.apache.airavata.mft.api.service.TransferApiRequest\x1a\x38.org.apache.airavata.mft.api.service.TransferApiResponse\x12\x92\x01\n\x13submitBatchTransfer\x12<.org.apache.airavata.mft.api.service.BatchTransferApiRequest\x1a=.org.apache.airavata.mft.api.service.BatchTransferApiResponse\x12\x89\x01\n\x10submitHttpUpload\x12\x39.org.apache.airavata.mft.api.service.HttpUploadApiRequest\x1a:.org.apache.airavata.mft.api.service.HttpUploadApiResponse\x12\x8f\x01\n\x12submitHttpDownload\x12;.org.apache.airavata.mft.api.service.HttpDownloadApiRequest\x1a<.org.apache.airavata.mft.api.service.HttpDownloadApiResponse\x12\x92\x01\n\x11getTransferStates\x12<.org.apache.airavata.mft.api.service.TransferStateApiRequest\x1a=.org.apache.airavata.mft.api.service.TransferStateApiResponse0\x01\x12\x8f\x01\n\x10getTransferState\x12<.org.apache.airavata.mft.api.service.TransferStateApiRequest\x1a=.org.apache.airavata.mft.api.service.TransferStateApiResponse\x12\x9f\x01\n\x17getResourceAvailability\x12\x41.org.apache.airavata.mft.api.service.FetchResourceMetadataRequest\x1a\x41.org.apache.airavata.mft.api.service.ResourceAvailabilityResponse\x12\x8b\x01\n\x10resourceMetadata\x12\x41.org.apache.airavata.mft.api.service.FetchResourceMetadataRequest\x1a\x34.org.apache.airavata.mft.agent.stub.ResourceMetadataB\x02P\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x14MFTTransferApi.proto\x12#org.apache.airavata.mft.api.service\x1a\x10\x43redCommon.proto\x1a\x13MFTAgentStubs.proto\"\x9b\x01\n\x10\x43\x61llbackEndpoint\x12P\n\x04type\x18\x01 \x01(\x0e\x32\x42.org.apache.airavata.mft.api.service.CallbackEndpoint.CallbackType\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\"#\n\x0c\x43\x61llbackType\x12\x08\n\x04HTTP\x10\x00\x12\t\n\x05KAFKA\x10\x01\"\xf3\x03\n\x12TransferApiRequest\x12\x12\n\nsourcePath\x18\x01 \x01(\t\x12\x17\n\x0fsourceStorageId\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x17\n\x0f\x64\x65stinationPath\x18\x04 \x01(\t\x12\x1c\n\x14\x64\x65stinationStorageId\x18\x05 \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\x06 \x01(\t\x12\x18\n\x10\x61\x66\x66inityTransfer\x18\x07 \x01(\x08\x12_\n\x0ctargetAgents\x18\x08 \x03(\x0b\x32I.org.apache.airavata.mft.api.service.TransferApiRequest.TargetAgentsEntry\x12H\n\x15mftAuthorizationToken\x18\t \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\x12P\n\x11\x63\x61llbackEndpoints\x18\n \x03(\x0b\x32\x35.org.apache.airavata.mft.api.service.CallbackEndpoint\x1a\x33\n\x11TargetAgentsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\")\n\x13TransferApiResponse\x12\x12\n\ntransferId\x18\x01 \x01(\t\"l\n\x17\x42\x61tchTransferApiRequest\x12Q\n\x10transferRequests\x18\x01 \x03(\x0b\x32\x37.org.apache.airavata.mft.api.service.TransferApiRequest\"/\n\x18\x42\x61tchTransferApiResponse\x12\x13\n\x0btransferIds\x18\x01 \x03(\t\"\xc3\x01\n\x14HttpUploadApiRequest\x12\x1c\n\x14\x64\x65stinationStorageId\x18\x01 \x01(\t\x12\x14\n\x0cresourcePath\x18\x02 \x01(\t\x12\x18\n\x10\x64\x65stinationToken\x18\x03 \x01(\t\x12\x13\n\x0btargetAgent\x18\x05 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x06 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\"9\n\x15HttpUploadApiResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0btargetAgent\x18\x02 \x01(\t\"\xbb\x01\n\x16HttpDownloadApiRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x17\n\x0fsourceStorageId\x18\x02 \x01(\t\x12\x13\n\x0bsourceToken\x18\x03 \x01(\t\x12\x13\n\x0btargetAgent\x18\x05 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x06 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\";\n\x17HttpDownloadApiResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x13\n\x0btargetAgent\x18\x02 \x01(\t\"w\n\x17TransferStateApiRequest\x12\x12\n\ntransferId\x18\x01 \x01(\t\x12H\n\x15mftAuthorizationToken\x18\x02 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthToken\"j\n\x18TransferStateApiResponse\x12\r\n\x05state\x18\x01 \x01(\t\x12\x16\n\x0eupdateTimeMils\x18\x02 \x01(\x03\x12\x12\n\npercentage\x18\x03 \x01(\x01\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\"1\n\x1cResourceAvailabilityResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\"w\n!GetResourceMetadataFromIDsRequest\x12\x14\n\x0cresourcePath\x18\x01 \x01(\t\x12\x11\n\tstorageId\x18\x02 \x01(\t\x12\x10\n\x08secretId\x18\x03 \x01(\t\x12\x17\n\x0frecursiveSearch\x18\x04 \x01(\x08\"\xa9\x02\n\x1c\x46\x65tchResourceMetadataRequest\x12W\n\rdirectRequest\x18\x01 \x01(\x0b\x32>.org.apache.airavata.mft.agent.stub.GetResourceMetadataRequestH\x00\x12[\n\tidRequest\x18\x02 \x01(\x0b\x32\x46.org.apache.airavata.mft.api.service.GetResourceMetadataFromIDsRequestH\x00\x12H\n\x15mftAuthorizationToken\x18\x03 \x01(\x0b\x32).org.apache.airavata.mft.common.AuthTokenB\t\n\x07request2\xa4\t\n\x12MFTTransferService\x12\x83\x01\n\x0esubmitTransfer\x12\x37.org.apache.airavata.mft.api.service.TransferApiRequest\x1a\x38.org.apache.airavata.mft.api.service.TransferApiResponse\x12\x92\x01\n\x13submitBatchTransfer\x12<.org.apache.airavata.mft.api.service.BatchTransferApiRequest\x1a=.org.apache.airavata.mft.api.service.BatchTransferApiResponse\x12\x89\x01\n\x10submitHttpUpload\x12\x39.org.apache.airavata.mft.api.service.HttpUploadApiRequest\x1a:.org.apache.airavata.mft.api.service.HttpUploadApiResponse\x12\x8f\x01\n\x12submitHttpDownload\x12;.org.apache.airavata.mft.api.service.HttpDownloadApiRequest\x1a<.org.apache.airavata.mft.api.service.HttpDownloadApiResponse\x12\x92\x01\n\x11getTransferStates\x12<.org.apache.airavata.mft.api.service.TransferStateApiRequest\x1a=.org.apache.airavata.mft.api.service.TransferStateApiResponse0\x01\x12\x8f\x01\n\x10getTransferState\x12<.org.apache.airavata.mft.api.service.TransferStateApiRequest\x1a=.org.apache.airavata.mft.api.service.TransferStateApiResponse\x12\x9f\x01\n\x17getResourceAvailability\x12\x41.org.apache.airavata.mft.api.service.FetchResourceMetadataRequest\x1a\x41.org.apache.airavata.mft.api.service.ResourceAvailabilityResponse\x12\x8b\x01\n\x10resourceMetadata\x12\x41.org.apache.airavata.mft.api.service.FetchResourceMetadataRequest\x1a\x34.org.apache.airavata.mft.agent.stub.ResourceMetadataB\x02P\x01\x62\x06proto3') @@ -178,9 +178,9 @@ _RESOURCEAVAILABILITYRESPONSE._serialized_start=1699 _RESOURCEAVAILABILITYRESPONSE._serialized_end=1748 _GETRESOURCEMETADATAFROMIDSREQUEST._serialized_start=1750 - _GETRESOURCEMETADATAFROMIDSREQUEST._serialized_end=1844 - _FETCHRESOURCEMETADATAREQUEST._serialized_start=1847 - _FETCHRESOURCEMETADATAREQUEST._serialized_end=2144 - _MFTTRANSFERSERVICE._serialized_start=2147 - _MFTTRANSFERSERVICE._serialized_end=3335 + _GETRESOURCEMETADATAFROMIDSREQUEST._serialized_end=1869 + _FETCHRESOURCEMETADATAREQUEST._serialized_start=1872 + _FETCHRESOURCEMETADATAREQUEST._serialized_end=2169 + _MFTTRANSFERSERVICE._serialized_start=2172 + _MFTTRANSFERSERVICE._serialized_end=3360 # @@protoc_insertion_point(module_scope) diff --git a/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2.py b/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2.py index de190084..2122e7d2 100644 --- a/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2.py +++ b/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2.py @@ -15,7 +15,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1a\x63ommon/StorageCommon.proto\x12\x35org.apache.airavata.mft.resource.stubs.storage.common\".\n\x19StorageTypeResolveRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"\xea\x01\n\x1aStorageTypeResolveResponse\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x13\n\x0bstorageName\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\x12K\n\x05\x65rror\x18\x04 \x01(\x0e\x32<.org.apache.airavata.mft.resource.stubs.storage.common.Error\"\xdd\x01\n\x10SecretForStorage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x10\n\x08secretId\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\x12K\n\x05\x65rror\x18\x04 \x01(\x0e\x32<.org.apache.airavata.mft.resource.stubs.storage.common.Error\"/\n\x1aSecretForStorageGetRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"2\n\x1dSecretForStorageDeleteRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"0\n\x1eSecretForStorageDeleteResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\x93\x01\n\x10StorageListEntry\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x13\n\x0bstorageName\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\"s\n\x13StorageListResponse\x12\\\n\x0bstorageList\x18\x01 \x03(\x0b\x32G.org.apache.airavata.mft.resource.stubs.storage.common.StorageListEntry\":\n\x12StorageListRequest\x12\x12\n\npageNumber\x18\x01 \x01(\x05\x12\x10\n\x08pageSize\x18\x02 \x01(\x05*r\n\x0bStorageType\x12\x06\n\x02S3\x10\x00\x12\x07\n\x03SCP\x10\x01\x12\x07\n\x03\x46TP\x10\x02\x12\t\n\x05LOCAL\x10\x03\x12\x07\n\x03\x42OX\x10\x04\x12\x0b\n\x07\x44ROPBOX\x10\x05\x12\x07\n\x03GCS\x10\x06\x12\t\n\x05\x41ZURE\x10\x07\x12\t\n\x05SWIFT\x10\x08\x12\t\n\x05ODATA\x10\t*?\n\x05\x45rror\x12\r\n\tNOT_FOUND\x10\x00\x12\x11\n\rNO_PERMISSION\x10\x01\x12\x14\n\x10LIMIT_OVERFLOWED\x10\x02\x32\xa6\x07\n\x14StorageCommonService\x12\xb9\x01\n\x12resolveStorageType\x12P.org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest\x1aQ.org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse\x12\xac\x01\n\x18registerSecretForStorage\x12G.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x1aG.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x12\xb1\x01\n\x13getSecretForStorage\x12Q.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageGetRequest\x1aG.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x12\xc6\x01\n\x17\x64\x65leteSecretsForStorage\x12T.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageDeleteRequest\x1aU.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageDeleteResponse\x12\xa5\x01\n\x0clistStorages\x12I.org.apache.airavata.mft.resource.stubs.storage.common.StorageListRequest\x1aJ.org.apache.airavata.mft.resource.stubs.storage.common.StorageListResponseB\x02P\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1a\x63ommon/StorageCommon.proto\x12\x35org.apache.airavata.mft.resource.stubs.storage.common\".\n\x19StorageTypeResolveRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"\xea\x01\n\x1aStorageTypeResolveResponse\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x13\n\x0bstorageName\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\x12K\n\x05\x65rror\x18\x04 \x01(\x0e\x32<.org.apache.airavata.mft.resource.stubs.storage.common.Error\"\xdd\x01\n\x10SecretForStorage\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x10\n\x08secretId\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\x12K\n\x05\x65rror\x18\x04 \x01(\x0e\x32<.org.apache.airavata.mft.resource.stubs.storage.common.Error\"/\n\x1aSecretForStorageGetRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"2\n\x1dSecretForStorageDeleteRequest\x12\x11\n\tstorageId\x18\x01 \x01(\t\"0\n\x1eSecretForStorageDeleteResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\x93\x01\n\x10StorageListEntry\x12\x11\n\tstorageId\x18\x01 \x01(\t\x12\x13\n\x0bstorageName\x18\x02 \x01(\t\x12W\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageType\"s\n\x13StorageListResponse\x12\\\n\x0bstorageList\x18\x01 \x03(\x0b\x32G.org.apache.airavata.mft.resource.stubs.storage.common.StorageListEntry\":\n\x12StorageListRequest\x12\x12\n\npageNumber\x18\x01 \x01(\x05\x12\x10\n\x08pageSize\x18\x02 \x01(\x05\"\xac\x01\n\x14StorageSearchRequest\x12\x13\n\tstorageId\x18\x01 \x01(\tH\x00\x12\x15\n\x0bstorageName\x18\x02 \x01(\tH\x00\x12Y\n\x0bstorageType\x18\x03 \x01(\x0e\x32\x42.org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeH\x00\x42\r\n\x0bsearchQuery*r\n\x0bStorageType\x12\x06\n\x02S3\x10\x00\x12\x07\n\x03SCP\x10\x01\x12\x07\n\x03\x46TP\x10\x02\x12\t\n\x05LOCAL\x10\x03\x12\x07\n\x03\x42OX\x10\x04\x12\x0b\n\x07\x44ROPBOX\x10\x05\x12\x07\n\x03GCS\x10\x06\x12\t\n\x05\x41ZURE\x10\x07\x12\t\n\x05SWIFT\x10\x08\x12\t\n\x05ODATA\x10\t*?\n\x05\x45rror\x12\r\n\tNOT_FOUND\x10\x00\x12\x11\n\rNO_PERMISSION\x10\x01\x12\x14\n\x10LIMIT_OVERFLOWED\x10\x02\x32\xd2\x08\n\x14StorageCommonService\x12\xb9\x01\n\x12resolveStorageType\x12P.org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest\x1aQ.org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse\x12\xac\x01\n\x18registerSecretForStorage\x12G.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x1aG.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x12\xb1\x01\n\x13getSecretForStorage\x12Q.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageGetRequest\x1aG.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorage\x12\xc6\x01\n\x17\x64\x65leteSecretsForStorage\x12T.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageDeleteRequest\x1aU.org.apache.airavata.mft.resource.stubs.storage.common.SecretForStorageDeleteResponse\x12\xa9\x01\n\x0esearchStorages\x12K.org.apache.airavata.mft.resource.stubs.storage.common.StorageSearchRequest\x1aJ.org.apache.airavata.mft.resource.stubs.storage.common.StorageListResponse\x12\xa5\x01\n\x0clistStorages\x12I.org.apache.airavata.mft.resource.stubs.storage.common.StorageListRequest\x1aJ.org.apache.airavata.mft.resource.stubs.storage.common.StorageListResponseB\x02P\x01\x62\x06proto3') _STORAGETYPE = DESCRIPTOR.enum_types_by_name['StorageType'] StorageType = enum_type_wrapper.EnumTypeWrapper(_STORAGETYPE) @@ -45,6 +45,7 @@ _STORAGELISTENTRY = DESCRIPTOR.message_types_by_name['StorageListEntry'] _STORAGELISTRESPONSE = DESCRIPTOR.message_types_by_name['StorageListResponse'] _STORAGELISTREQUEST = DESCRIPTOR.message_types_by_name['StorageListRequest'] +_STORAGESEARCHREQUEST = DESCRIPTOR.message_types_by_name['StorageSearchRequest'] StorageTypeResolveRequest = _reflection.GeneratedProtocolMessageType('StorageTypeResolveRequest', (_message.Message,), { 'DESCRIPTOR' : _STORAGETYPERESOLVEREQUEST, '__module__' : 'common.StorageCommon_pb2' @@ -108,15 +109,22 @@ }) _sym_db.RegisterMessage(StorageListRequest) +StorageSearchRequest = _reflection.GeneratedProtocolMessageType('StorageSearchRequest', (_message.Message,), { + 'DESCRIPTOR' : _STORAGESEARCHREQUEST, + '__module__' : 'common.StorageCommon_pb2' + # @@protoc_insertion_point(class_scope:org.apache.airavata.mft.resource.stubs.storage.common.StorageSearchRequest) + }) +_sym_db.RegisterMessage(StorageSearchRequest) + _STORAGECOMMONSERVICE = DESCRIPTOR.services_by_name['StorageCommonService'] if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'P\001' - _STORAGETYPE._serialized_start=1072 - _STORAGETYPE._serialized_end=1186 - _ERROR._serialized_start=1188 - _ERROR._serialized_end=1251 + _STORAGETYPE._serialized_start=1247 + _STORAGETYPE._serialized_end=1361 + _ERROR._serialized_start=1363 + _ERROR._serialized_end=1426 _STORAGETYPERESOLVEREQUEST._serialized_start=85 _STORAGETYPERESOLVEREQUEST._serialized_end=131 _STORAGETYPERESOLVERESPONSE._serialized_start=134 @@ -135,6 +143,8 @@ _STORAGELISTRESPONSE._serialized_end=1010 _STORAGELISTREQUEST._serialized_start=1012 _STORAGELISTREQUEST._serialized_end=1070 - _STORAGECOMMONSERVICE._serialized_start=1254 - _STORAGECOMMONSERVICE._serialized_end=2188 + _STORAGESEARCHREQUEST._serialized_start=1073 + _STORAGESEARCHREQUEST._serialized_end=1245 + _STORAGECOMMONSERVICE._serialized_start=1429 + _STORAGECOMMONSERVICE._serialized_end=2535 # @@protoc_insertion_point(module_scope) diff --git a/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2_grpc.py b/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2_grpc.py index 75bbed02..4ee8ad14 100644 --- a/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2_grpc.py +++ b/python-sdk/src/airavata_mft_sdk/common/StorageCommon_pb2_grpc.py @@ -34,6 +34,11 @@ def __init__(self, channel): request_serializer=common_dot_StorageCommon__pb2.SecretForStorageDeleteRequest.SerializeToString, response_deserializer=common_dot_StorageCommon__pb2.SecretForStorageDeleteResponse.FromString, ) + self.searchStorages = channel.unary_unary( + '/org.apache.airavata.mft.resource.stubs.storage.common.StorageCommonService/searchStorages', + request_serializer=common_dot_StorageCommon__pb2.StorageSearchRequest.SerializeToString, + response_deserializer=common_dot_StorageCommon__pb2.StorageListResponse.FromString, + ) self.listStorages = channel.unary_unary( '/org.apache.airavata.mft.resource.stubs.storage.common.StorageCommonService/listStorages', request_serializer=common_dot_StorageCommon__pb2.StorageListRequest.SerializeToString, @@ -68,6 +73,12 @@ def deleteSecretsForStorage(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def searchStorages(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def listStorages(self, request, context): """Missing associated documentation comment in .proto file.""" context.set_code(grpc.StatusCode.UNIMPLEMENTED) @@ -97,6 +108,11 @@ def add_StorageCommonServiceServicer_to_server(servicer, server): request_deserializer=common_dot_StorageCommon__pb2.SecretForStorageDeleteRequest.FromString, response_serializer=common_dot_StorageCommon__pb2.SecretForStorageDeleteResponse.SerializeToString, ), + 'searchStorages': grpc.unary_unary_rpc_method_handler( + servicer.searchStorages, + request_deserializer=common_dot_StorageCommon__pb2.StorageSearchRequest.FromString, + response_serializer=common_dot_StorageCommon__pb2.StorageListResponse.SerializeToString, + ), 'listStorages': grpc.unary_unary_rpc_method_handler( servicer.listStorages, request_deserializer=common_dot_StorageCommon__pb2.StorageListRequest.FromString, @@ -180,6 +196,23 @@ def deleteSecretsForStorage(request, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod + def searchStorages(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/org.apache.airavata.mft.resource.stubs.storage.common.StorageCommonService/searchStorages', + common_dot_StorageCommon__pb2.StorageSearchRequest.SerializeToString, + common_dot_StorageCommon__pb2.StorageListResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + @staticmethod def listStorages(request, target, diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java index 8f1b4511..8664f03a 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/ResourceBackend.java @@ -38,7 +38,7 @@ public interface ResourceBackend { public SecretForStorage getSecretForStorage(SecretForStorageGetRequest request) throws Exception; public SecretForStorage registerSecretForStorage(SecretForStorage request) throws Exception; public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) throws Exception; - + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception; public StorageListResponse listStorage(StorageListRequest request) throws Exception; public SCPStorageListResponse listSCPStorage(SCPStorageListRequest request) throws Exception; public Optional getSCPStorage(SCPStorageGetRequest request) throws Exception; diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java index 28e84766..ec9067c0 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/SQLResourceBackend.java @@ -109,6 +109,7 @@ public SecretForStorage registerSecretForStorage(SecretForStorage request) throw ety.setSecretId(request.getSecretId()); ety.setStorageId(request.getStorageId()); ety.setType(request.getStorageType().name()); + resourceSecretRepository.save(ety); return request; } @@ -118,6 +119,44 @@ public boolean deleteSecretForStorage(SecretForStorageDeleteRequest request) thr return true; } + @Override + public StorageListResponse searchStorages(StorageSearchRequest request) throws Exception { + StorageListResponse.Builder resp = StorageListResponse.newBuilder(); + switch (request.getSearchQueryCase()) { + case STORAGEID: + Optional storageOp = resolveStorageRepository.getByStorageId(request.getStorageId()); + if (storageOp.isPresent()) { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(storageOp.get().getStorageId()); + entry.setStorageName(storageOp.get().getStorageName()); + entry.setStorageType(StorageType.valueOf(storageOp.get().getStorageType().name())); + resp.addStorageList(entry); + } + break; + case STORAGENAME: + List storages = resolveStorageRepository.getByStorageName(request.getStorageName()); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + case STORAGETYPE: + storages = resolveStorageRepository.getByStorageType(ResolveStorageEntity.StorageType.valueOf(request.getStorageType().name())); + storages.forEach(st -> { + StorageListEntry.Builder entry = StorageListEntry.newBuilder(); + entry.setStorageId(st.getStorageId()); + entry.setStorageName(st.getStorageName()); + entry.setStorageType(StorageType.valueOf(st.getStorageType().name())); + resp.addStorageList(entry); + }); + break; + } + return resp.build(); + } + @Override public StorageListResponse listStorage(StorageListRequest request) throws Exception { Iterable all = resolveStorageRepository.findAll(); diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ResolveStorageRepository.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ResolveStorageRepository.java index 3ee60aa0..59827e63 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ResolveStorageRepository.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/backend/sql/repository/ResolveStorageRepository.java @@ -20,8 +20,12 @@ import org.apache.airavata.mft.resource.server.backend.sql.entity.ResolveStorageEntity; import org.springframework.data.repository.CrudRepository; +import java.util.Collection; +import java.util.List; import java.util.Optional; public interface ResolveStorageRepository extends CrudRepository { public Optional getByStorageId(String storageID); + public List getByStorageName(String storageName); + public List getByStorageType(ResolveStorageEntity.StorageType storageType); } diff --git a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/StorageCommonServiceHandler.java b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/StorageCommonServiceHandler.java index d54c9afc..c5de59e3 100644 --- a/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/StorageCommonServiceHandler.java +++ b/services/resource-service/server/src/main/java/org/apache/airavata/mft/resource/server/handler/StorageCommonServiceHandler.java @@ -99,6 +99,21 @@ public void deleteSecretsForStorage(SecretForStorageDeleteRequest request, Strea } } + @Override + public void searchStorages(StorageSearchRequest request, StreamObserver responseObserver) { + try { + StorageListResponse storageListResponse = this.backend.searchStorages(request); + responseObserver.onNext(storageListResponse); + responseObserver.onCompleted(); + } catch (Exception e) { + logger.error("Failed searching storages", e); + + responseObserver.onError(Status.INTERNAL.withCause(e) + .withDescription("Failed listing storages\"") + .asRuntimeException()); + } + } + @Override public void listStorages(StorageListRequest request, StreamObserver responseObserver) { try { diff --git a/services/resource-service/stub/src/main/proto/common/StorageCommon.proto b/services/resource-service/stub/src/main/proto/common/StorageCommon.proto index d10f713b..3f62f6d0 100644 --- a/services/resource-service/stub/src/main/proto/common/StorageCommon.proto +++ b/services/resource-service/stub/src/main/proto/common/StorageCommon.proto @@ -84,10 +84,19 @@ message StorageListRequest { int32 pageSize = 2; } +message StorageSearchRequest { + oneof searchQuery { + string storageId = 1; + string storageName = 2; + StorageType storageType = 3; + } +} + service StorageCommonService { rpc resolveStorageType (StorageTypeResolveRequest) returns (StorageTypeResolveResponse); rpc registerSecretForStorage(SecretForStorage) returns (SecretForStorage); rpc getSecretForStorage(SecretForStorageGetRequest) returns (SecretForStorage); rpc deleteSecretsForStorage(SecretForStorageDeleteRequest) returns (SecretForStorageDeleteResponse); + rpc searchStorages(StorageSearchRequest) returns (StorageListResponse); rpc listStorages(StorageListRequest) returns (StorageListResponse); } \ No newline at end of file diff --git a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java index 79f87159..5b08c809 100644 --- a/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java +++ b/transport/azure-transport/src/main/java/org/apache/airavata/mft/transport/azure/AzureMetadataCollector.java @@ -50,7 +50,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); // Azure does not have a concept called hierarchical containers. So we assume that there are no containers inside diff --git a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java index 48e6cb89..526111cf 100644 --- a/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java +++ b/transport/box-transport/src/main/java/org/apache/airavata/mft/transport/box/BoxMetadataCollector.java @@ -48,7 +48,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); diff --git a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java index 1a767e87..c9dbb64c 100644 --- a/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java +++ b/transport/dropbox-transport/src/main/java/org/apache/airavata/mft/transport/dropbox/DropboxMetadataCollector.java @@ -50,7 +50,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); DbxRequestConfig config = DbxRequestConfig.newBuilder("mftdropbox/v1").build(); diff --git a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java index 2e2ba333..31e434bb 100644 --- a/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java +++ b/transport/ftp-transport/src/main/java/org/apache/airavata/mft/transport/ftp/FTPMetadataCollector.java @@ -48,7 +48,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder(); diff --git a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java index df32a6f5..44f04748 100644 --- a/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java +++ b/transport/gcp-transport/src/main/java/org/apache/airavata/mft/transport/gcp/GCSMetadataCollector.java @@ -51,7 +51,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); PrivateKey privKey = GCSUtil.getPrivateKey(gcsSecret.getPrivateKey()); diff --git a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java index e6a6f1b5..a25d8d7f 100644 --- a/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java +++ b/transport/local-transport/src/main/java/org/apache/airavata/mft/transport/local/LocalMetadataCollector.java @@ -62,7 +62,7 @@ private FileMetadata.Builder getFileBuilderFromPath(File file) throws Exception return fileBuilder; } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); diff --git a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java index 818df459..5b3252ad 100644 --- a/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java +++ b/transport/odata-transport/src/main/java/org/apache/airavata/mft/transport/odata/ODataMetadataCollector.java @@ -65,7 +65,7 @@ private CloseableHttpClient getHttpClient(ODataSecret oDataSecret) { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder(); Optional fileResourceMetadata = findFileResourceMetadata(resourcePath); if (fileResourceMetadata.isPresent()) { diff --git a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java index 998cb1d3..f0af5b6f 100644 --- a/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java +++ b/transport/s3-transport/src/main/java/org/apache/airavata/mft/transport/s3/S3MetadataCollector.java @@ -34,7 +34,10 @@ import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage; import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; public class S3MetadataCollector implements MetadataCollector { @@ -56,7 +59,7 @@ private void checkInitialized() { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); @@ -96,48 +99,92 @@ public ResourceMetadata getResourceMetadata(String resourcePath) throws Exceptio return resourceBuilder.build(); } + // If directory path or top level bucket path + if (resourcePath.endsWith("/") || resourcePath.isEmpty()) { // Directory + ObjectListing objectListing = s3Client.listObjects(s3Storage.getBucketName(), resourcePath); + resourceBuilder.setDirectory(processDirectory(resourcePath, objectListing)); + + } else if (s3Client.doesObjectExist(s3Storage.getBucketName(), resourcePath)){ // File - String key = s3Client.getObject(s3Storage.getBucketName(), resourcePath).getKey(); - - if (key.endsWith("/")) { // Folder - - ObjectListing objectListing = s3Client.listObjects(s3Storage.getBucketName(), key); - List objectSummaries = objectListing.getObjectSummaries(); - DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder(); - for (S3ObjectSummary summary: objectSummaries) { - if (summary.getKey().endsWith("/")) { - DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder(); - subDirBuilder.setCreatedTime(summary.getLastModified().getTime()); - subDirBuilder.setUpdateTime(summary.getLastModified().getTime()); - subDirBuilder.setResourcePath(summary.getKey()); - subDirBuilder.setFriendlyName(new File(summary.getKey()).getName()); - dirBuilder.addDirectories(subDirBuilder); - } else { - FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); - fileBuilder.setUpdateTime(summary.getLastModified().getTime()); - fileBuilder.setCreatedTime(summary.getLastModified().getTime()); - fileBuilder.setResourcePath(summary.getKey()); - fileBuilder.setFriendlyName(new File(summary.getKey()).getName()); - fileBuilder.setResourceSize(summary.getSize()); - dirBuilder.addFiles(fileBuilder); - } - } - resourceBuilder.setDirectory(dirBuilder); - } else { // File FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); ObjectMetadata fileMetadata = s3Client.getObjectMetadata(s3Storage.getBucketName(), resourcePath); fileBuilder.setResourceSize(fileMetadata.getContentLength()); fileBuilder.setResourcePath(resourcePath); - fileBuilder.setMd5Sum(fileMetadata.getContentMD5()); + fileBuilder.setMd5Sum(fileMetadata.getContentMD5() == null ? "" : fileMetadata.getContentMD5() ); fileBuilder.setFriendlyName(new File(resourcePath).getName()); fileBuilder.setCreatedTime(fileMetadata.getLastModified().getTime()); fileBuilder.setUpdateTime(fileMetadata.getLastModified().getTime()); resourceBuilder.setFile(fileBuilder); + } else { // Try if user forgot add trailing / + ObjectListing objectListing = s3Client.listObjects(s3Storage.getBucketName(), resourcePath + "/"); + resourceBuilder.setDirectory(processDirectory(resourcePath + "/", objectListing)); } return resourceBuilder.build(); } + private DirectoryMetadata.Builder processDirectory(String resourcePath, ObjectListing objectListing) { + + List objectSummaries = objectListing.getObjectSummaries(); + + Map subDirCache = new HashMap<>(); + Map> childTree = new HashMap<>(); + childTree.put(resourcePath, new ArrayList<>()); + + DirectoryMetadata.Builder dirBuilder = DirectoryMetadata.newBuilder(); + subDirCache.put(resourcePath, dirBuilder); + + for (S3ObjectSummary summary: objectSummaries) { + buildStructureRecursively(resourcePath, summary.getKey(), summary, subDirCache, childTree); + } + + registerChildren(resourcePath, subDirCache, childTree); + + return dirBuilder; + } + + private void registerChildren(String parentPath, Map directoryStore, + Map> childTree) { + for (String childDir : childTree.get(parentPath)) { + registerChildren(childDir, directoryStore, childTree); + directoryStore.get(parentPath).addDirectories(directoryStore.get(childDir)); + } + } + + private void buildStructureRecursively(String basePath, String filePath, S3ObjectSummary summary, + Map directoryStore, + Map> childTree) { + String relativePath = filePath.substring(basePath.length()); + if (relativePath.contains("/")) { // A Directory + String[] pathSections = relativePath.split("/"); + + String thisDirKey = basePath + pathSections[0] + "/"; + + if (!directoryStore.containsKey(thisDirKey)) { + DirectoryMetadata.Builder subDirBuilder = DirectoryMetadata.newBuilder(); + subDirBuilder.setCreatedTime(summary.getLastModified().getTime()); + subDirBuilder.setUpdateTime(summary.getLastModified().getTime()); + subDirBuilder.setResourcePath(thisDirKey); + subDirBuilder.setFriendlyName(pathSections[0]); + directoryStore.put(thisDirKey, subDirBuilder); + childTree.get(basePath).add(thisDirKey); + childTree.put(thisDirKey, new ArrayList<>()); + } + + //directoryStore.get(basePath).addDirectories(subDirBuilder); + buildStructureRecursively(thisDirKey, filePath, summary, directoryStore, childTree); + + } else { // A File + FileMetadata.Builder fileBuilder = FileMetadata.newBuilder(); + fileBuilder.setUpdateTime(summary.getLastModified().getTime()); + fileBuilder.setCreatedTime(summary.getLastModified().getTime()); + fileBuilder.setResourcePath(summary.getKey()); + fileBuilder.setFriendlyName(new File(summary.getKey()).getName()); + fileBuilder.setResourceSize(summary.getSize()); + directoryStore.get(basePath).addFiles(fileBuilder); + } + } + @Override public Boolean isAvailable(String resourcePath) throws Exception { diff --git a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java index b937875e..9048b152 100644 --- a/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java +++ b/transport/scp-transport/src/main/java/org/apache/airavata/mft/transport/scp/SCPMetadataCollector.java @@ -57,7 +57,7 @@ public void init(StorageWrapper storage, SecretWrapper secret) { } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { ResourceMetadata.Builder resourceBuilder = ResourceMetadata.newBuilder(); try (SSHClient sshClient = getSSHClient(scpStorage, scpSecret)) { diff --git a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java index f2648522..b9f33e3f 100644 --- a/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java +++ b/transport/swift-transport/src/main/java/org/apache/airavata/mft/transport/swift/SwiftMetadataCollector.java @@ -79,7 +79,7 @@ private SwiftApi getSwiftApi(SwiftStorage swiftStorage, SwiftSecret swiftSecret) } @Override - public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception { + public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception { checkInitialized(); SwiftApi swiftApi = getSwiftApi(swiftStorage, swiftSecret);