Skip to content

Commit

Permalink
Support to list storages through CLI
Browse files Browse the repository at this point in the history
  • Loading branch information
DImuthuUpe committed Dec 25, 2022
1 parent 27b9d68 commit 8124fa2
Show file tree
Hide file tree
Showing 30 changed files with 491 additions and 74 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public void processTransfer(String transferId, AgentTransferRequest request,
MetadataCollector srcMetadataCollector = srcMetadataCollectorOp.orElseThrow(() -> new Exception("Could not find a metadata collector for source"));
srcMetadataCollector.init(request.getSourceStorage(), request.getSourceSecret());

ResourceMetadata srcMetadata = srcMetadataCollector.getResourceMetadata(request.getSourcePath());
ResourceMetadata srcMetadata = srcMetadataCollector.getResourceMetadata(request.getSourcePath(), false);
if (srcMetadata.getMetadataCase() != ResourceMetadata.MetadataCase.FILE) {
throw new Exception("Expected a file as the source but received " + srcMetadata.getMetadataCase().name());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ public String resolveRPCRequest(SyncRPCRequest request) throws Exception {
if (metadataCollectorOptional.isPresent()) {
MetadataCollector metadataCollector = metadataCollectorOptional.get();
metadataCollector.init(req.getStorage(), req.getSecret());
ResourceMetadata resourceMetadata = metadataCollector.getResourceMetadata(req.getResourcePath());
ResourceMetadata resourceMetadata = metadataCollector.getResourceMetadata(req.getResourcePath(), directResourceMetadataReq.getRecursiveSearch());
return JsonFormat.printer().print(resourceMetadata);
} else {
throw new Exception("No metadata collector for type " + req.getStorage().getStorageCase().name());
Expand Down
1 change: 1 addition & 0 deletions agent/stub/src/main/proto/MFTAgentStubs.proto
Original file line number Diff line number Diff line change
Expand Up @@ -114,5 +114,6 @@ message GetResourceMetadataRequest {
string resourcePath = 1;
StorageWrapper storage = 2;
SecretWrapper secret = 3;
bool recursiveSearch = 4;
}

12 changes: 7 additions & 5 deletions api/service/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,15 @@
</exclusions>
</dependency>

<!-- To be removed -->

<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-scp-transport</artifactId>
<version>0.01-SNAPSHOT</version>
<scope>runtime</scope>
<artifactId>mft-resource-service-client</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.airavata</groupId>
<artifactId>mft-secret-service-client</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,50 @@
import org.apache.airavata.mft.admin.models.rpc.SyncRPCResponse;
import org.apache.airavata.mft.agent.stub.*;
import org.apache.airavata.mft.api.service.*;
import org.apache.airavata.mft.credential.stubs.azure.AzureSecret;
import org.apache.airavata.mft.credential.stubs.azure.AzureSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.box.BoxSecret;
import org.apache.airavata.mft.credential.stubs.box.BoxSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecret;
import org.apache.airavata.mft.credential.stubs.dropbox.DropboxSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecret;
import org.apache.airavata.mft.credential.stubs.ftp.FTPSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.gcs.GCSSecret;
import org.apache.airavata.mft.credential.stubs.gcs.GCSSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.odata.ODataSecret;
import org.apache.airavata.mft.credential.stubs.odata.ODataSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.s3.S3Secret;
import org.apache.airavata.mft.credential.stubs.s3.S3SecretGetRequest;
import org.apache.airavata.mft.credential.stubs.scp.SCPSecret;
import org.apache.airavata.mft.credential.stubs.scp.SCPSecretGetRequest;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecret;
import org.apache.airavata.mft.credential.stubs.swift.SwiftSecretGetRequest;
import org.apache.airavata.mft.resource.client.StorageServiceClient;
import org.apache.airavata.mft.resource.client.StorageServiceClientBuilder;
import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorage;
import org.apache.airavata.mft.resource.stubs.azure.storage.AzureStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorage;
import org.apache.airavata.mft.resource.stubs.box.storage.BoxStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorage;
import org.apache.airavata.mft.resource.stubs.dropbox.storage.DropboxStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorage;
import org.apache.airavata.mft.resource.stubs.ftp.storage.FTPStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorage;
import org.apache.airavata.mft.resource.stubs.gcs.storage.GCSStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorage;
import org.apache.airavata.mft.resource.stubs.local.storage.LocalStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorage;
import org.apache.airavata.mft.resource.stubs.odata.storage.ODataStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3Storage;
import org.apache.airavata.mft.resource.stubs.s3.storage.S3StorageGetRequest;
import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorage;
import org.apache.airavata.mft.resource.stubs.scp.storage.SCPStorageGetRequest;
import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveRequest;
import org.apache.airavata.mft.resource.stubs.storage.common.StorageTypeResolveResponse;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorage;
import org.apache.airavata.mft.resource.stubs.swift.storage.SwiftStorageGetRequest;
import org.apache.airavata.mft.secret.client.SecretServiceClient;
import org.apache.airavata.mft.secret.client.SecretServiceClientBuilder;
import org.apache.commons.lang3.tuple.Pair;
import org.dozer.DozerBeanMapper;
import org.lognet.springboot.grpc.GRpcService;
Expand Down Expand Up @@ -243,10 +287,151 @@ public void getTransferState(TransferStateApiRequest request, StreamObserver<Tra
}
}

private GetResourceMetadataRequest deriveDirectRequest(GetResourceMetadataFromIDsRequest idRequest) {


StorageServiceClient storageClient = StorageServiceClientBuilder.buildClient(resourceServiceHost, resourceServicePort);
SecretServiceClient secretClient = SecretServiceClientBuilder.buildClient(secretServiceHost, secretServicePort);
StorageTypeResolveResponse storageTypeResp = storageClient.common().resolveStorageType(
StorageTypeResolveRequest.newBuilder().setStorageId(idRequest.getStorageId()).build());

GetResourceMetadataRequest.Builder directReqBuilder = GetResourceMetadataRequest.newBuilder();
directReqBuilder.setResourcePath(idRequest.getResourcePath());
directReqBuilder.setRecursiveSearch(idRequest.getRecursiveSearch());

switch (storageTypeResp.getStorageType()) {
case S3:
S3Storage s3Storage = storageClient.s3()
.getS3Storage(S3StorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
S3Secret s3Secret = secretClient.s3()
.getS3Secret(S3SecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setS3(s3Storage).build())
.setSecret(SecretWrapper.newBuilder().setS3(s3Secret).build());
break;
case FTP:
FTPStorage ftpStorage = storageClient.ftp()
.getFTPStorage(FTPStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
FTPSecret ftpSecret = secretClient.ftp()
.getFTPSecret(FTPSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setFtp(ftpStorage).build())
.setSecret(SecretWrapper.newBuilder().setFtp(ftpSecret).build());
break;
case LOCAL:
LocalStorage localStorage = storageClient.local()
.getLocalStorage(LocalStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setLocal(localStorage).build());
break;
case BOX:
BoxStorage boxStorage = storageClient.box()
.getBoxStorage(BoxStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
BoxSecret boxSecret = secretClient.box()
.getBoxSecret(BoxSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setBox(boxStorage).build())
.setSecret(SecretWrapper.newBuilder().setBox(boxSecret).build());
break;
case DROPBOX:
DropboxStorage dropBoxStorage = storageClient.dropbox()
.getDropboxStorage(DropboxStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
DropboxSecret dropBoxSecret = secretClient.dropbox()
.getDropboxSecret(DropboxSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setDropbox(dropBoxStorage).build())
.setSecret(SecretWrapper.newBuilder().setDropbox(dropBoxSecret).build());
break;
case GCS:
GCSStorage gcsStorage = storageClient.gcs()
.getGCSStorage(GCSStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
GCSSecret gcsSecret = secretClient.gcs()
.getGCSSecret(GCSSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setGcs(gcsStorage).build())
.setSecret(SecretWrapper.newBuilder().setGcs(gcsSecret).build());
break;
case AZURE:
AzureStorage azureStorage = storageClient.azure()
.getAzureStorage(AzureStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
AzureSecret azureSecret = secretClient.azure()
.getAzureSecret(AzureSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setAzure(azureStorage).build())
.setSecret(SecretWrapper.newBuilder().setAzure(azureSecret).build());
break;
case SWIFT:
SwiftStorage swiftStorage = storageClient.swift()
.getSwiftStorage(SwiftStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
SwiftSecret swiftSecret = secretClient.swift()
.getSwiftSecret(SwiftSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setSwift(swiftStorage).build())
.setSecret(SecretWrapper.newBuilder().setSwift(swiftSecret).build());
break;
case ODATA:
ODataStorage odataStorage = storageClient.odata()
.getODataStorage(ODataStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
ODataSecret odataSecret = secretClient.odata()
.getODataSecret(ODataSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setOdata(odataStorage).build())
.setSecret(SecretWrapper.newBuilder().setOdata(odataSecret).build());
break;
case SCP:
SCPStorage scpStorage = storageClient.scp()
.getSCPStorage(SCPStorageGetRequest.newBuilder()
.setStorageId(idRequest.getStorageId()).build());
SCPSecret scpSecret = secretClient.scp()
.getSCPSecret(SCPSecretGetRequest.newBuilder()
.setSecretId(idRequest.getSecretId()).build());

directReqBuilder
.setStorage(StorageWrapper.newBuilder().setScp(scpStorage).build())
.setSecret(SecretWrapper.newBuilder().setScp(scpSecret).build());
break;

}

return directReqBuilder.build();
}
@Override
public void getResourceAvailability(FetchResourceMetadataRequest request, StreamObserver<ResourceAvailabilityResponse> responseObserver) {
GetResourceMetadataRequest directRequest = request.getDirectRequest();
GetResourceMetadataRequest directRequest = null;

try {
if (request.getRequestCase() == FetchResourceMetadataRequest.RequestCase.DIRECTREQUEST) {
directRequest = request.getDirectRequest();
} else {
directRequest = deriveDirectRequest(request.getIdRequest());
}

String targetAgent = derriveTargetAgent("");
SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder()
.withAgentId(targetAgent)
Expand Down Expand Up @@ -282,8 +467,16 @@ public void getResourceAvailability(FetchResourceMetadataRequest request, Stream
@Override
public void resourceMetadata(FetchResourceMetadataRequest request, StreamObserver<ResourceMetadata> responseObserver) {

GetResourceMetadataRequest directRequest = request.getDirectRequest();
GetResourceMetadataRequest directRequest = null;

try {

if (request.getRequestCase() == FetchResourceMetadataRequest.RequestCase.DIRECTREQUEST) {
directRequest = request.getDirectRequest();
} else {
directRequest = deriveDirectRequest(request.getIdRequest());
}

String targetAgent = derriveTargetAgent("");
SyncRPCRequest.SyncRPCRequestBuilder requestBuilder = SyncRPCRequest.SyncRPCRequestBuilder.builder()
.withAgentId(targetAgent)
Expand Down
1 change: 1 addition & 0 deletions api/stub/src/main/proto/MFTTransferApi.proto
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ message GetResourceMetadataFromIDsRequest {
string resourcePath = 1;
string storageId = 2;
string secretId = 3;
bool recursiveSearch = 4;
}

message FetchResourceMetadataRequest {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import org.apache.airavata.mft.agent.stub.ResourceMetadata;
import org.apache.airavata.mft.agent.stub.SecretWrapper;
import org.apache.airavata.mft.agent.stub.StorageWrapper;
import org.apache.airavata.mft.core.DirectoryResourceMetadata;
import org.apache.airavata.mft.core.FileResourceMetadata;

public interface MetadataCollector {
Expand All @@ -36,11 +35,12 @@ public interface MetadataCollector {
/**
* Fetches a metadata of given File Resource
*
* @param resourcePath path of the resource
* @param resourcePath path of the resource
* @param recursiveSearch
* @return an object of {@link FileResourceMetadata}
* @throws Exception if the resource id is not a File Resource type or the resource can't be fetched from the resource service
*/
public ResourceMetadata getResourceMetadata(String resourcePath) throws Exception;
public ResourceMetadata getResourceMetadata(String resourcePath, boolean recursiveSearch) throws Exception;

/**
* Check whether the resource is available in the actual storage
Expand Down
2 changes: 1 addition & 1 deletion python-cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ Install dependencies
```
pip install grpcio==1.46.3
pip install grpcio-tools==1.46.3
pip install airavata_mft_sdk==0.0.1-alpha15
pip install airavata_mft_sdk==0.0.1-alpha18
```

Build the binary
Expand Down
65 changes: 64 additions & 1 deletion python-cli/mft_cli/mft_cli/main.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,74 @@
import typer
import mft_cli.storage
from airavata_mft_sdk import mft_client
from airavata_mft_sdk.common import StorageCommon_pb2
from airavata_mft_sdk import MFTTransferApi_pb2
from rich.console import Console
from rich.table import Table

app = typer.Typer()

storage_app = typer.Typer()
app.add_typer(mft_cli.storage.app, name="storage")

@app.command("ls")
def list(storage_path):
storage_name = storage_path.split("/")[0]
resource_path = storage_path[len(storage_name) +1 :]
client = mft_client.MFTClient()
search_req = StorageCommon_pb2.StorageSearchRequest(storageName=storage_name)
storages = client.common_api.searchStorages(search_req)

if len(storages.storageList) == 0:
search_req = StorageCommon_pb2.StorageSearchRequest(storageId=storage_name)
storages = client.common_api.searchStorages(search_req)

if len(storages.storageList) == 0:
print("No storage with name or id " + storage_name + " was found. Please register the storage with command mft-cli storage add")
exit()

if len(storages.storageList) > 1:
print("More than one storage with nam " + storage_name + " was found. Please use the storage id. You can fetch it from mft-cli storage list")
exit()

storage = storages.storageList[0]
sec_req = StorageCommon_pb2.SecretForStorageGetRequest(storageId = storage.storageId)
sec_resp = client.common_api.getSecretForStorage(sec_req)
if sec_resp.error != 0:
print("Could not fetch the secret for storage " + storage.storageId)

id_req = MFTTransferApi_pb2.GetResourceMetadataFromIDsRequest(storageId = sec_resp.storageId,
secretId = sec_resp.secretId,
resourcePath = resource_path)
resource_medata_req = MFTTransferApi_pb2.FetchResourceMetadataRequest(idRequest = id_req)

metadata_resp = client.transfer_api.resourceMetadata(resource_medata_req)

console = Console()
table = Table()

table.add_column('Name', justify='left')
table.add_column('Type', justify='center')
table.add_column('Size', justify='center')

if (metadata_resp.WhichOneof('metadata') == 'directory') :
for dir in metadata_resp.directory.directories:
table.add_row('[bold]' + dir.friendlyName + '[/bold]', 'DIR', '')

for file in metadata_resp.directory.files:
table.add_row('[bold]' + file.friendlyName + '[/bold]', 'FILE', str(file.resourceSize))

elif (metadata_resp.WhichOneof('metadata') == 'file'):
table.add_row('[bold]' + metadata_resp.file.friendlyName + '[/bold]', 'FILE', str(metadata_resp.file.resourceSize))

elif (metadata_resp.WhichOneof('metadata') == 'error'):
print(metadata_resp.error)

console.print(table)

@app.command("cp")
def copy(source, destination):
print("Moving data from " + source + " to " + destination)


if __name__ == "__main__":
app()
Loading

0 comments on commit 8124fa2

Please sign in to comment.