Skip to content

Commit

Permalink
Merge branch 'uoe'
Browse files Browse the repository at this point in the history
  • Loading branch information
@dspeed2
dspeed2 committed Feb 19, 2018
2 parents 62edad5 + 21d354a commit 699c3cc
Show file tree
Hide file tree
Showing 6 changed files with 279 additions and 277 deletions.
17 changes: 17 additions & 0 deletions datavault-common/src/main/java/org/datavaultplatform/common/storage/Device.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package org.datavaultplatform.common.storage;

import org.datavaultplatform.common.io.Progress;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.io.File;

Expand All @@ -10,6 +13,8 @@ public abstract class Device {

// Some public information about a device or storage system
public String name;
public Boolean multipleCopies = false;
public List<String> locations = null;

// Some private configuration properties
protected Map<String,String> config;
Expand All @@ -26,7 +31,19 @@ public Device(String name, Map<String,String> config) {
// Progress information should be updated for monitoring as the copy occurs
public abstract void retrieve(String path, File working, Progress progress) throws Exception;

// This method should be overridden by plugins that allow the retrieval of multiple copies to get a specific copy
public void retrieve(String path, File working, Progress progress, String location) throws Exception {
throw new UnsupportedOperationException();
}
// Copy an object (file/dir) from the working space
// Progress information should be updated for monitoring as the copy occurs
public abstract String store(String path, File working, Progress progress) throws Exception;

public Boolean hasMultipleCopies() {
return this.multipleCopies;
}

public List<String> getLocations() {
return this.locations;
}
}
221 changes: 43 additions & 178 deletions ...-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
package org.datavaultplatform.common.storage.impl;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Map;
import java.util.UUID;

Expand All @@ -17,23 +16,24 @@
public class TivoliStorageManager extends Device implements ArchiveStore {

private static final Logger logger = LoggerFactory.getLogger(TivoliStorageManager.class);
public static final String TSM_SERVER_NODE1_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm1.opt";
public static final String TSM_SERVER_NODE2_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm2.opt";
//public List<String> locations = null;

// todo : can we change this to COPY_BACK?
public Verify.Method verificationMethod = Verify.Method.LOCAL_ONLY;

public TivoliStorageManager(String name, Map<String,String> config) throws Exception {
super(name, config);
locations = new ArrayList<String>();
locations.add(TivoliStorageManager.TSM_SERVER_NODE1_OPT);
locations.add(TivoliStorageManager.TSM_SERVER_NODE2_OPT);
super.multipleCopies = true;

// Unpack the config parameters (in an implementation-specific way)
// Actually I can't think of any parameters that we need.
}

// @Override
// public long getUsableSpace() throws Exception {
// File file = new File(rootPath);
// return file.getUsableSpace();
// }

@Override
public long getUsableSpace() throws Exception {
long retVal = 0;
Expand All @@ -59,18 +59,6 @@ public long getUsableSpace() throws Exception {

return retVal;
}

// @Override
// public void retrieve(String path, File working, Progress progress) throws Exception {
// Path absolutePath = getAbsolutePath(path);
// File file = absolutePath.toFile();
//
// if (file.isFile()) {
// FileCopy.copyFile(progress, file, working);
// } else if (file.isDirectory()) {
// FileCopy.copyDirectory(progress, file, working);
// }
// }

@Override
public void retrieve(String path, File working, Progress progress) throws Exception {
Expand All @@ -83,69 +71,79 @@ public void retrieve(String path, File working, Progress progress) throws Except
// in the local version of this class the FileCopy class adds info to the progess object
// I don't think we need to use the patch at all in this version

logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath());
ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath());

//try {
// if this fails try to retrieve from second node, I think the only failure possible
// at this stage is a missing archive, checksums are checked later in Retrieve.java
// this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT);
//} catch (Exception node1) {
// if this fails too just continue on to the error display
// this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT);
//}
throw new UnsupportedOperationException();
}

@Override
public void retrieve(String path, File working, Progress progress, String optFilePath) throws Exception {

logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + optFilePath);

ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + optFilePath, "-replace=true");
Process p = pb.start();

// This class is already running in its own thread so it can happily pause until finished.
p.waitFor();

if (p.exitValue() != 0) {
logger.info("Retrieval of " + working.getName() + " failed. ");
logger.info("Retrieval of " + working.getName() + " failed using " + optFilePath + ". ");
InputStream error = p.getErrorStream();
for (int i = 0; i < error.available(); i++) {
logger.info("" + error.read());
}
throw new Exception("Retrieval of " + working.getName() + " failed. ");

}
// FILL IN THE REST OF PROGRESS x dirs, x files, x bytes etc.
}

// @Override
// public String store(String path, File working, Progress progress) throws Exception {
// Path absolutePath = getAbsolutePath(path);
// File retrieveFile = absolutePath.resolve(working.getName()).toFile();
//
// if (working.isFile()) {
// FileCopy.copyFile(progress, working, retrieveFile);
// } else if (working.isDirectory()) {
// FileCopy.copyDirectory(progress, working, retrieveFile);
// }
//
// return working.getName();
// }

@Override
public String store(String path, File working, Progress progress) throws Exception {



// todo : monitor progress

// Note: generate a uuid to be passed as the description. We should probably use the deposit UUID instead (do we need a specialised archive method)?
// Just a thought - Does the filename contain the deposit uuid? Could we use that as the description?
String randomUUIDString = UUID.randomUUID().toString();

this.storeInTSMNode(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT, randomUUIDString);
this.storeInTSMNode(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT, randomUUIDString);

return randomUUIDString;
}

private String storeInTSMNode(String path, File working, Progress progress, String optFilePath, String description) throws Exception {

// check we have enough space to store the data (is the file bagged and tarred atm or is the actual space going to be different?)
// actually the Deposit / Retreive worker classes check the free space it appears if we get here we don't need to check

// The working file appears to be bagged and tarred when we get here
// in the local version of this class the FileCopy class adds info to the progess object
// I don't think we need to use the patch at all in this version
logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + randomUUIDString);
ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + randomUUIDString);
logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + description + " -optfile=" + optFilePath);
ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + description, "-optfile=" + optFilePath);

Process p = pb.start();

// This class is already running in its own thread so it can happily pause until finished.
p.waitFor();

if (p.exitValue() != 0) {
logger.info("Deposit of " + working.getName() + " failed. ");
logger.info("Deposit of " + working.getName() + " using " + optFilePath + " failed. ");
logger.info(p.getErrorStream().toString());
logger.info(p.getOutputStream().toString());
throw new Exception("Deposit of " + working.getName() + " failed. ");
throw new Exception("Deposit of " + working.getName() + " using " + optFilePath + " failed. ");
}

return randomUUIDString;
return description;
}


Expand All @@ -154,137 +152,4 @@ public String store(String path, File working, Progress progress) throws Excepti
public Verify.Method getVerifyMethod() {
return verificationMethod;
}

// @Override
// public long getUsableSpaceToolBox() throws Exception {
// long retVal = 0;
//
// ITSMSession session = null;
// session = this.getTSMSession();
// IFilespace filespace = session.retrieveFilespace(FILESPACE_NAME);
// retVal = filespace.getCapacity();
// this.closeTSMSession(session);
// return retVal;
// }

// private ITSMSession getTSMSession() throws ToolboxException{
// ITSMSession session = null;
//try {
// // Initializing a single-threaded environment
// Toolbox.initSingleThreading();
//
// // Connecting to the server with custom connection options
// ConnectionOptions connectionOptions = new ConnectionOptions();
// connectionOptions.setConfigFile(CONFIG_FILE);
// session = Toolbox.createSession(connectionOptions);
//
// System.out.println("Connecting successful.");
//} catch (ToolboxException e) {
// Throwable cause = e.getCause();
// if (cause instanceof TSMException
// && ((TSMException) cause).getReturnCode() == TsmReturnCodes.DSM_RC_NO_PASS_FILE) {
// System.out.println("No password file, generating one.");
//
// ConnectionOptions connectionOptions = new ConnectionOptions();
// connectionOptions.setConfigFile(CONFIG_FILE);
// connectionOptions.setPassword(PASSWORD);
// session = Toolbox.createSession(connectionOptions);
// } else {
// e.printStackTrace();
// throw e;
// }
//}
//
//return session;
//}

//private void closeTSMSession(ITSMSession session) {
// if (session) {
// // Disconnecting
// session.disconnect();
// System.out.println("Disconnecting successful.");
//
// // Cleaning up the environment
// Toolbox.cleanUp();
// }
//}
//
//public String storeToolbox(String path, File working, Progress progress) throws Exception {
//
//// todo : monitor progress
//
//// Note: generate a uuid to be passed as the description. We should probably use the deposit UUID instead (do we need a specialised archive method)?
//// Just a thought - Does the filename contain the deposit uuid? Could we use that as the description?
//String randomUUIDString = UUID.randomUUID().toString();
//String NODE_NAME = "TOOLBOX";
//String PASSWORD = "PASSWORD";
////String INPUT_FILE_NAME = "inputs/archive.txt";
//String OWNER = "Owner";
//String LOW_LEVEL_NAME = "/OBJECT1";
//String HIGH_LEVEL_NAME = "/ARCHIVE";
//String FILESPACE_NAME = "/ARCHIVEFS";
//ITSMSession session = null;
//try {
// session = this.getTSMSession();
//
// // Registering the filespace
// IFilespace filespace = session.retrieveFilespace(FILESPACE_NAME);
//
// // Checking if filespace exists
// if (filespace == null) {
// System.out.printf("Warning: '%s' does not exists.\n", FILESPACE_NAME);
// return;
// }
//
// // Creating a new archive object
// IArchiveObject archive = filespace.createArchiveFile(
// HIGH_LEVEL_NAME,
// LOW_LEVEL_NAME,
// OWNER);
// archive.setDescription(archive.getLowLevelName());
//
// try {
// // Binding the management class
// archive.bindManagementClass();
// } catch (ToolboxException e) {
// // Checking if archive copy group exists in the bound management class
// if (e.getReturnCode() == ToolboxErrorCodes.TSM_EXCEPTION ) {
// TSMException ex = (TSMException) e.getCause();
// if (ex.getReturnCode() == TsmReturnCodes.DSM_RC_TL_NOACG) {
// System.out.println(ex.getMessage());
// return;
// }
// }
// return;
// }
// File file = new File(INPUT_FILE_NAME);
// FileInputStream in = new FileInputStream(file);
//
// // Setting object size
// archive.setEstimatedSize(file.length());
//
// // Beginning a transaction
// session.beginTransaction();
// try {
// // Sending the object to the server
// archive.send(in);
// session.commit();
// System.out.printf("Archive object '%s%s%S' has been sent.\n",
// FILESPACE_NAME, HIGH_LEVEL_NAME, LOW_LEVEL_NAME);
// } catch(ToolboxException e) {
// // Rolling back the current transaction
// session.rollback();
// }
//} catch (ToolboxException e) {
// e.printStackTrace();
//} finally {
// this.closeTSMSession(session);
//}
//
//return randomUUIDString;
//}

// public void retrieveToolBox(String path, File working, Progress progress) throws Exception {
//
//}
}
Loading

0 comments on commit 699c3cc

Please sign in to comment.