From 7c96b94d2fb531f8fb818c96b0f0f72dd674f68a Mon Sep 17 00:00:00 2001 From: David Speed Date: Thu, 1 Feb 2018 16:31:17 +0000 Subject: [PATCH 1/5] IssueId #402 Made a start on refactoring things so that duplicate copies of the vault are stored in TSM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1) Updated TivoliStorageManager.java so that it now has two hardcoded strings to dsm1.opt and dsm2.opt (currently hardcoded to a location I have the files in, we’ll need to make sure the deployment scripts copy these to the correct location) 2) Updated TivoliStorageManager.java so that the TSM store / retrieve client calls now use the optfile=dsm1.opt (which uses the current node) 3) Removed a lot of commented out stuff --- .../storage/impl/TivoliStorageManager.java | 179 +----------------- .../src/test/resources/tsm/dsm1.opt | 19 ++ .../src/test/resources/tsm/dsm2.opt | 19 ++ 3 files changed, 46 insertions(+), 171 deletions(-) create mode 100644 datavault-worker/src/test/resources/tsm/dsm1.opt create mode 100644 datavault-worker/src/test/resources/tsm/dsm2.opt diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java index ab357c0b1..8c00a1936 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java @@ -17,6 +17,8 @@ public class TivoliStorageManager extends Device implements ArchiveStore { private static final Logger logger = LoggerFactory.getLogger(TivoliStorageManager.class); + private static final String TSM_SERVER_NODE1_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm1.opt"; + private static final String TSM_SERVER_NODE2_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm2.opt"; // todo : can we change this to COPY_BACK? public Verify.Method verificationMethod = Verify.Method.LOCAL_ONLY; @@ -28,12 +30,6 @@ public TivoliStorageManager(String name, Map config) throws Excep // Actually I can't think of any parameters that we need. } -// @Override -// public long getUsableSpace() throws Exception { -// File file = new File(rootPath); -// return file.getUsableSpace(); -// } - @Override public long getUsableSpace() throws Exception { long retVal = 0; @@ -59,18 +55,6 @@ public long getUsableSpace() throws Exception { return retVal; } - -// @Override -// public void retrieve(String path, File working, Progress progress) throws Exception { -// Path absolutePath = getAbsolutePath(path); -// File file = absolutePath.toFile(); -// -// if (file.isFile()) { -// FileCopy.copyFile(progress, file, working); -// } else if (file.isDirectory()) { -// FileCopy.copyDirectory(progress, file, working); -// } -// } @Override public void retrieve(String path, File working, Progress progress) throws Exception { @@ -83,9 +67,8 @@ public void retrieve(String path, File working, Progress progress) throws Except // in the local version of this class the FileCopy class adds info to the progess object // I don't think we need to use the patch at all in this version - logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath()); - ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath()); - + logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); + ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); Process p = pb.start(); // This class is already running in its own thread so it can happily pause until finished. @@ -100,24 +83,11 @@ public void retrieve(String path, File working, Progress progress) throws Except throw new Exception("Retrieval of " + working.getName() + " failed. "); } } - -// @Override -// public String store(String path, File working, Progress progress) throws Exception { -// Path absolutePath = getAbsolutePath(path); -// File retrieveFile = absolutePath.resolve(working.getName()).toFile(); -// -// if (working.isFile()) { -// FileCopy.copyFile(progress, working, retrieveFile); -// } else if (working.isDirectory()) { -// FileCopy.copyDirectory(progress, working, retrieveFile); -// } -// -// return working.getName(); -// } @Override public String store(String path, File working, Progress progress) throws Exception { - + + // todo : monitor progress // Note: generate a uuid to be passed as the description. We should probably use the deposit UUID instead (do we need a specialised archive method)? @@ -130,8 +100,8 @@ public String store(String path, File working, Progress progress) throws Excepti // The working file appears to be bagged and tarred when we get here // in the local version of this class the FileCopy class adds info to the progess object // I don't think we need to use the patch at all in this version - logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + randomUUIDString); - ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + randomUUIDString); + logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + randomUUIDString + " -optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); + ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + randomUUIDString, "-optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); Process p = pb.start(); @@ -154,137 +124,4 @@ public String store(String path, File working, Progress progress) throws Excepti public Verify.Method getVerifyMethod() { return verificationMethod; } - -// @Override -// public long getUsableSpaceToolBox() throws Exception { -// long retVal = 0; -// -// ITSMSession session = null; -// session = this.getTSMSession(); -// IFilespace filespace = session.retrieveFilespace(FILESPACE_NAME); -// retVal = filespace.getCapacity(); -// this.closeTSMSession(session); -// return retVal; -// } - -// private ITSMSession getTSMSession() throws ToolboxException{ -// ITSMSession session = null; -//try { -// // Initializing a single-threaded environment -// Toolbox.initSingleThreading(); -// -// // Connecting to the server with custom connection options -// ConnectionOptions connectionOptions = new ConnectionOptions(); -// connectionOptions.setConfigFile(CONFIG_FILE); -// session = Toolbox.createSession(connectionOptions); -// -// System.out.println("Connecting successful."); -//} catch (ToolboxException e) { -// Throwable cause = e.getCause(); -// if (cause instanceof TSMException -// && ((TSMException) cause).getReturnCode() == TsmReturnCodes.DSM_RC_NO_PASS_FILE) { -// System.out.println("No password file, generating one."); -// -// ConnectionOptions connectionOptions = new ConnectionOptions(); -// connectionOptions.setConfigFile(CONFIG_FILE); -// connectionOptions.setPassword(PASSWORD); -// session = Toolbox.createSession(connectionOptions); -// } else { -// e.printStackTrace(); -// throw e; -// } -//} -// -//return session; -//} - -//private void closeTSMSession(ITSMSession session) { -// if (session) { -// // Disconnecting -// session.disconnect(); -// System.out.println("Disconnecting successful."); -// -// // Cleaning up the environment -// Toolbox.cleanUp(); -// } -//} -// -//public String storeToolbox(String path, File working, Progress progress) throws Exception { -// -//// todo : monitor progress -// -//// Note: generate a uuid to be passed as the description. We should probably use the deposit UUID instead (do we need a specialised archive method)? -//// Just a thought - Does the filename contain the deposit uuid? Could we use that as the description? -//String randomUUIDString = UUID.randomUUID().toString(); -//String NODE_NAME = "TOOLBOX"; -//String PASSWORD = "PASSWORD"; -////String INPUT_FILE_NAME = "inputs/archive.txt"; -//String OWNER = "Owner"; -//String LOW_LEVEL_NAME = "/OBJECT1"; -//String HIGH_LEVEL_NAME = "/ARCHIVE"; -//String FILESPACE_NAME = "/ARCHIVEFS"; -//ITSMSession session = null; -//try { -// session = this.getTSMSession(); -// -// // Registering the filespace -// IFilespace filespace = session.retrieveFilespace(FILESPACE_NAME); -// -// // Checking if filespace exists -// if (filespace == null) { -// System.out.printf("Warning: '%s' does not exists.\n", FILESPACE_NAME); -// return; -// } -// -// // Creating a new archive object -// IArchiveObject archive = filespace.createArchiveFile( -// HIGH_LEVEL_NAME, -// LOW_LEVEL_NAME, -// OWNER); -// archive.setDescription(archive.getLowLevelName()); -// -// try { -// // Binding the management class -// archive.bindManagementClass(); -// } catch (ToolboxException e) { -// // Checking if archive copy group exists in the bound management class -// if (e.getReturnCode() == ToolboxErrorCodes.TSM_EXCEPTION ) { -// TSMException ex = (TSMException) e.getCause(); -// if (ex.getReturnCode() == TsmReturnCodes.DSM_RC_TL_NOACG) { -// System.out.println(ex.getMessage()); -// return; -// } -// } -// return; -// } -// File file = new File(INPUT_FILE_NAME); -// FileInputStream in = new FileInputStream(file); -// -// // Setting object size -// archive.setEstimatedSize(file.length()); -// -// // Beginning a transaction -// session.beginTransaction(); -// try { -// // Sending the object to the server -// archive.send(in); -// session.commit(); -// System.out.printf("Archive object '%s%s%S' has been sent.\n", -// FILESPACE_NAME, HIGH_LEVEL_NAME, LOW_LEVEL_NAME); -// } catch(ToolboxException e) { -// // Rolling back the current transaction -// session.rollback(); -// } -//} catch (ToolboxException e) { -// e.printStackTrace(); -//} finally { -// this.closeTSMSession(session); -//} -// -//return randomUUIDString; -//} - -// public void retrieveToolBox(String path, File working, Progress progress) throws Exception { -// -//} } diff --git a/datavault-worker/src/test/resources/tsm/dsm1.opt b/datavault-worker/src/test/resources/tsm/dsm1.opt new file mode 100644 index 000000000..f89587baa --- /dev/null +++ b/datavault-worker/src/test/resources/tsm/dsm1.opt @@ -0,0 +1,19 @@ +************************************************************************ +* IBM Tivoli Storage Manager * +* * +* Sample Client User Options file for UNIX (dsm.opt.smp) * +************************************************************************ + +* This file contains an option you can use to specify the TSM +* server to contact if more than one is defined in your client +* system options file (dsm.sys). Copy dsm.opt.smp to dsm.opt. +* If you enter a server name for the option below, remove the +* leading asterisk (*). + +************************************************************************ + +* SErvername A server name defined in the dsm.sys file +* compressalways yes +nodename DLIB_SIDLAWS +quiet + diff --git a/datavault-worker/src/test/resources/tsm/dsm2.opt b/datavault-worker/src/test/resources/tsm/dsm2.opt new file mode 100644 index 000000000..5a6127721 --- /dev/null +++ b/datavault-worker/src/test/resources/tsm/dsm2.opt @@ -0,0 +1,19 @@ +************************************************************************ +* IBM Tivoli Storage Manager * +* * +* Sample Client User Options file for UNIX (dsm.opt.smp) * +************************************************************************ + +* This file contains an option you can use to specify the TSM +* server to contact if more than one is defined in your client +* system options file (dsm.sys). Copy dsm.opt.smp to dsm.opt. +* If you enter a server name for the option below, remove the +* leading asterisk (*). + +************************************************************************ + +* SErvername A server name defined in the dsm.sys file +* compressalways yes +virtualnodename DLIB-SIDLAWS2 +quiet + From 969b3d999c9bf05a9f2b7c516daa70fe9f2b8978 Mon Sep 17 00:00:00 2001 From: David Speed Date: Tue, 6 Feb 2018 10:22:13 +0000 Subject: [PATCH 2/5] IssueId #402 Updated the TSM store command so that two copies are stored in both my test nodes 1) Updated TivoliStorageManger to have some new consts to define the TSM nodes (this will be moved into config at some point) 2) Split the store method into store and storeInTSMNode so the store signature stays the same to comply with the interface but we can reuse the majority of the method for each node. 3) Updated TivoliStorageManager so that the store tests now expect the second node copy. 4) Added template dsm1.opt and dsm2.opt files which we can use with any deployment scripts to ensure the node config is correct --- .../storage/impl/TivoliStorageManager.java | 22 ++++++--- .../impl/TivoliStorageManagerTest.java | 49 +++++++++++++------ .../src/test/resources/tsm/dsm1.opt | 1 - .../src/test/resources/tsm/dsm2.opt | 1 + 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java index 8c00a1936..4dcbdf4a0 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java @@ -17,8 +17,8 @@ public class TivoliStorageManager extends Device implements ArchiveStore { private static final Logger logger = LoggerFactory.getLogger(TivoliStorageManager.class); - private static final String TSM_SERVER_NODE1_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm1.opt"; - private static final String TSM_SERVER_NODE2_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm2.opt"; + public static final String TSM_SERVER_NODE1_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm1.opt"; + public static final String TSM_SERVER_NODE2_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm2.opt"; // todo : can we change this to COPY_BACK? public Verify.Method verificationMethod = Verify.Method.LOCAL_ONLY; @@ -94,14 +94,22 @@ public String store(String path, File working, Progress progress) throws Excepti // Just a thought - Does the filename contain the deposit uuid? Could we use that as the description? String randomUUIDString = UUID.randomUUID().toString(); + this.storeInTSMNode(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT, randomUUIDString); + this.storeInTSMNode(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT, randomUUIDString); + + return randomUUIDString; + } + + public String storeInTSMNode(String path, File working, Progress progress, String optFilePath, String description) throws Exception { + // check we have enough space to store the data (is the file bagged and tarred atm or is the actual space going to be different?) // actually the Deposit / Retreive worker classes check the free space it appears if we get here we don't need to check // The working file appears to be bagged and tarred when we get here // in the local version of this class the FileCopy class adds info to the progess object // I don't think we need to use the patch at all in this version - logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + randomUUIDString + " -optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); - ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + randomUUIDString, "-optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); + logger.info("Store command is " + "dsmc" + " archive " + working.getAbsolutePath() + " -description=" + description + " -optfile=" + optFilePath); + ProcessBuilder pb = new ProcessBuilder("dsmc", "archive", working.getAbsolutePath(), "-description=" + description, "-optfile=" + optFilePath); Process p = pb.start(); @@ -109,13 +117,13 @@ public String store(String path, File working, Progress progress) throws Excepti p.waitFor(); if (p.exitValue() != 0) { - logger.info("Deposit of " + working.getName() + " failed. "); + logger.info("Deposit of " + working.getName() + " using " + optFilePath + " failed. "); logger.info(p.getErrorStream().toString()); logger.info(p.getOutputStream().toString()); - throw new Exception("Deposit of " + working.getName() + " failed. "); + throw new Exception("Deposit of " + working.getName() + " using " + optFilePath + " failed. "); } - return randomUUIDString; + return description; } diff --git a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java index 895bd5a4a..1a20fd3d1 100644 --- a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java +++ b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java @@ -46,8 +46,8 @@ public void setUp() { } } - private BufferedReader queryArchive(File working, String retVal) throws Exception{ - ProcessBuilder pb = new ProcessBuilder("dsmc", "query", "archive", working.getAbsolutePath(), "-description=" + retVal); + private BufferedReader queryArchive(File working, String description, String optFilePath) throws Exception{ + ProcessBuilder pb = new ProcessBuilder("dsmc", "query", "archive", working.getAbsolutePath(), "-description=" + description, "-optfile=" + optFilePath); Process p = pb.start(); p.waitFor(); @@ -57,10 +57,10 @@ private BufferedReader queryArchive(File working, String retVal) throws Exceptio return reader; } - private void deleteArchive(String retVal) throws Exception{ + private void deleteArchive(String description, String optFilePath) throws Exception{ String path = "/"; - ProcessBuilder pb = new ProcessBuilder("dsmc", "delete", "archive", path, "-subdir=yes", "-description=" + retVal, "-noprompt"); + ProcessBuilder pb = new ProcessBuilder("dsmc", "delete", "archive", path, "-subdir=yes", "-description=" + description, "-optfile=" + optFilePath, "-noprompt"); //System.out.println(pb.command()); Process p = pb.start(); @@ -77,14 +77,22 @@ public void testStoreValidParams() { // // store the tar // retVal = tsm.store(path, working, progress); // assertNotNull("RetVal should not be null", retVal); -// // check it is now in TSM -// BufferedReader reader = this.queryArchive(working, retVal); +// // check it is now in TSM node 1 +// BufferedReader reader = this.queryArchive(working, retVal, TivoliStorageManager.TSM_SERVER_NODE1_OPT); // String line = null; // while ( (line = reader.readLine()) != null) { -// assertFalse("Attempt to store failed", line.contains("No files matching search criteria were found")); +// assertFalse("Node1 attempt to store failed", line.contains("No files matching search criteria were found")); // } -// // delete from TSM -// deleteArchive(retVal); +// +// reader = this.queryArchive(working, retVal, TivoliStorageManager.TSM_SERVER_NODE2_OPT); +// line = null; +// while ( (line = reader.readLine()) != null) { +// assertFalse("Node2 attempt to store failed", line.contains("No files matching search criteria were found")); +// } +// +// // delete from TSM node 1 +// deleteArchive(retVal, TivoliStorageManager.TSM_SERVER_NODE1_OPT); +// deleteArchive(retVal, TivoliStorageManager.TSM_SERVER_NODE2_OPT); // } catch (Exception e) { // fail("Unexpected exception " + e.getMessage()); // } @@ -92,7 +100,7 @@ public void testStoreValidParams() { @Test public void testStoreNonExistantFile() { -// Progress progress = new Progress(); +// Progress progress = new Progress(); // File working = new File(tsmResources, "testx.tar"); // String path = "/tmp"; // try { @@ -102,7 +110,7 @@ public void testStoreNonExistantFile() { // fail("Exception should have been thrown"); // } catch (Exception e) { // assertNotNull("Exception should not be null", e); -// assertEquals("Message not as expected", "Deposit of testx.tar failed. ", e.getMessage()); +// assertEquals("Message not as expected", "Deposit of testx.tar using /opt/tivoli/tsm/client/ba/bin/dsm1.opt failed. ", e.getMessage()); // } } @@ -119,12 +127,18 @@ public void testRetrieveValidParams() { // retVal = tsm.store(path, working, progress); // //File retrieved = new File("/tmp/" + retVal); // assertNotNull("RetVal should not be null", retVal); -// // check it is now in TSM +// // check it is now in TSM node 1 // System.out.println("check it has been archived"); -// BufferedReader reader = this.queryArchive(working, retVal); +// BufferedReader reader = this.queryArchive(working, retVal, TivoliStorageManager.TSM_SERVER_NODE1_OPT); // String line = null; // while ( (line = reader.readLine()) != null) { -// assertFalse("Attempt to store failed", line.contains("No files matching search criteria were found")); +// assertFalse("Node 1 attempt to store failed", line.contains("No files matching search criteria were found")); +// } +// +// reader = this.queryArchive(working, retVal, TivoliStorageManager.TSM_SERVER_NODE2_OPT); +// line = null; +// while ( (line = reader.readLine()) != null) { +// assertFalse("Node 2 attempt to store failed", line.contains("No files matching search criteria were found")); // } // // // temp move the file (so we can retrieve without overwriting) @@ -140,9 +154,12 @@ public void testRetrieveValidParams() { // System.out.println("Check the retrieve archive exits"); // assertTrue("The retrieved archive doesn't exist", working.exists()); // -// // delete from TSM +// // delete from TSM node 1 // System.out.println("Delete from TSM"); -// deleteArchive(retVal); +// deleteArchive(retVal, TivoliStorageManager.TSM_SERVER_NODE1_OPT); +// +// // delete from TSM node2 +// deleteArchive(retVal, TivoliStorageManager.TSM_SERVER_NODE2_OPT); // // // delete the retrieved tar // System.out.println("Delete the retrieved archive"); diff --git a/datavault-worker/src/test/resources/tsm/dsm1.opt b/datavault-worker/src/test/resources/tsm/dsm1.opt index f89587baa..b729919ac 100644 --- a/datavault-worker/src/test/resources/tsm/dsm1.opt +++ b/datavault-worker/src/test/resources/tsm/dsm1.opt @@ -14,6 +14,5 @@ * SErvername A server name defined in the dsm.sys file * compressalways yes -nodename DLIB_SIDLAWS quiet diff --git a/datavault-worker/src/test/resources/tsm/dsm2.opt b/datavault-worker/src/test/resources/tsm/dsm2.opt index 5a6127721..493978fee 100644 --- a/datavault-worker/src/test/resources/tsm/dsm2.opt +++ b/datavault-worker/src/test/resources/tsm/dsm2.opt @@ -15,5 +15,6 @@ * SErvername A server name defined in the dsm.sys file * compressalways yes virtualnodename DLIB-SIDLAWS2 +password k5jg352 quiet From 68ebb30da69729d5846fb6b4c63c250c4c0e69d8 Mon Sep 17 00:00:00 2001 From: David Speed Date: Mon, 12 Feb 2018 11:20:59 +0000 Subject: [PATCH 3/5] IssueId #402 Updated the TSM retrieve command so that if the first TSM copy is missing the second one is used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1) Updated TivoliStorageManager.retrieve so that if we are not able to retrieve the first copy we just output a message to the logs and silently attempt to retrieve the 2nd copy. If that fails too another message will go to the logs and the usual on screen error messages will display. I’ve updated the existing unit tests to work with these changes but haven’t yet added new ones (I’ve tests this manually by deleting the first copy then trying to retrieve) I haven’t done anything about emailing folk when this happens. I’m going to make the changes to use the second copy if the checksums match before doing anything about the tests or warnings. --- .../storage/impl/TivoliStorageManager.java | 21 +++++++++++++++---- .../impl/TivoliStorageManagerTest.java | 2 +- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java index 4dcbdf4a0..48cc4affb 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java @@ -67,20 +67,33 @@ public void retrieve(String path, File working, Progress progress) throws Except // in the local version of this class the FileCopy class adds info to the progess object // I don't think we need to use the patch at all in this version - logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); - ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + TivoliStorageManager.TSM_SERVER_NODE1_OPT); + try { + // if this fails try to retrieve from second node, I think the only failure possible + // at this stage is a missing archive, checksums are checked later in Retrieve.java + this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT); + } catch (Exception node1) { + // if this fails too just continue on to the error display + this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT); + } + } + + public void retrieve(String path, File working, Progress progress, String optFilePath) throws Exception { + + logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + optFilePath); + ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + optFilePath); Process p = pb.start(); // This class is already running in its own thread so it can happily pause until finished. p.waitFor(); if (p.exitValue() != 0) { - logger.info("Retrieval of " + working.getName() + " failed. "); + logger.info("Retrieval of " + working.getName() + " failed using " + optFilePath + ". "); InputStream error = p.getErrorStream(); for (int i = 0; i < error.available(); i++) { logger.info("" + error.read()); } throw new Exception("Retrieval of " + working.getName() + " failed. "); + } } @@ -100,7 +113,7 @@ public String store(String path, File working, Progress progress) throws Excepti return randomUUIDString; } - public String storeInTSMNode(String path, File working, Progress progress, String optFilePath, String description) throws Exception { + private String storeInTSMNode(String path, File working, Progress progress, String optFilePath, String description) throws Exception { // check we have enough space to store the data (is the file bagged and tarred atm or is the actual space going to be different?) // actually the Deposit / Retreive worker classes check the free space it appears if we get here we don't need to check diff --git a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java index 1a20fd3d1..8a1426fff 100644 --- a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java +++ b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java @@ -145,7 +145,7 @@ public void testRetrieveValidParams() { // FileUtils.moveFile(working, temp); // assertFalse("The working file shouldn't exist", working.exists()); // -// // retrieve from TSM +// // retrieve from TSM node 1 // System.out.println("Retrieve the tar"); // //dsmc retrieve /tmp/datavault/temp/2848@ubuntu-xenial/513c2b11-30df-4947-846b-a64309c61eb8.tar // tsm.retrieve(path, working, progress); From e8ce658a4f95da84c3126ce6796e3a444cf0e3cc Mon Sep 17 00:00:00 2001 From: David Speed Date: Tue, 13 Feb 2018 09:51:18 +0000 Subject: [PATCH 4/5] IssueId #402 Updated the TSM retrieve command so that if the first TSM copies checksum is wrong we try again using the second copy I did a decent amount of refactoring on this yesterday. I've changed things so that the abstract Device class now has multipleCopies and locations members which allows us to identify whether a device has multiple copies and where the config is for these locations (this is all very TSM specific atm so will probably change, at the very least the locations will probably come from a config file rather than being hardcoded in the TSMManager class). I've also updated the retrieve task to use these changes, I've done limited testing so far but depositing / retrieving seems to work as expected atm (though I've done nothing more complicated than checking if a 'normal' (i.e. 1st copy present with matching checksum) retrieve works). I still need to check if the existing unit tests pass and add new ones. For the store task this still works the same as before i.e all the multiple stuff is done in the TSMManager so the new Device members are unused here. I might change this just so everything matches (if it makes sense and we have time) I'll hopefully manage to come back to this tomorrow. --- .../common/storage/Device.java | 17 ++ .../storage/impl/TivoliStorageManager.java | 21 +- .../worker/tasks/Retrieve.java | 190 ++++++++++-------- 3 files changed, 141 insertions(+), 87 deletions(-) diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/Device.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/Device.java index 3ffcd03dc..16bc46677 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/Device.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/Device.java @@ -1,6 +1,9 @@ package org.datavaultplatform.common.storage; import org.datavaultplatform.common.io.Progress; + +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.io.File; @@ -10,6 +13,8 @@ public abstract class Device { // Some public information about a device or storage system public String name; + public Boolean multipleCopies = false; + public List locations = null; // Some private configuration properties protected Map config; @@ -26,7 +31,19 @@ public Device(String name, Map config) { // Progress information should be updated for monitoring as the copy occurs public abstract void retrieve(String path, File working, Progress progress) throws Exception; + // This method should be overridden by plugins that allow the retrieval of multiple copies to get a specific copy + public void retrieve(String path, File working, Progress progress, String location) throws Exception { + throw new UnsupportedOperationException(); + } // Copy an object (file/dir) from the working space // Progress information should be updated for monitoring as the copy occurs public abstract String store(String path, File working, Progress progress) throws Exception; + + public Boolean hasMultipleCopies() { + return this.multipleCopies; + } + + public List getLocations() { + return this.locations; + } } diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java index 48cc4affb..d54bff388 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java @@ -1,9 +1,8 @@ package org.datavaultplatform.common.storage.impl; import java.io.File; -import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; +import java.util.ArrayList; import java.util.Map; import java.util.UUID; @@ -19,12 +18,17 @@ public class TivoliStorageManager extends Device implements ArchiveStore { private static final Logger logger = LoggerFactory.getLogger(TivoliStorageManager.class); public static final String TSM_SERVER_NODE1_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm1.opt"; public static final String TSM_SERVER_NODE2_OPT = "/opt/tivoli/tsm/client/ba/bin/dsm2.opt"; + //public List locations = null; // todo : can we change this to COPY_BACK? public Verify.Method verificationMethod = Verify.Method.LOCAL_ONLY; public TivoliStorageManager(String name, Map config) throws Exception { super(name, config); + locations = new ArrayList(); + locations.add(TivoliStorageManager.TSM_SERVER_NODE1_OPT); + locations.add(TivoliStorageManager.TSM_SERVER_NODE2_OPT); + super.multipleCopies = true; // Unpack the config parameters (in an implementation-specific way) // Actually I can't think of any parameters that we need. @@ -67,14 +71,15 @@ public void retrieve(String path, File working, Progress progress) throws Except // in the local version of this class the FileCopy class adds info to the progess object // I don't think we need to use the patch at all in this version - try { + //try { // if this fails try to retrieve from second node, I think the only failure possible // at this stage is a missing archive, checksums are checked later in Retrieve.java - this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT); - } catch (Exception node1) { + // this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT); + //} catch (Exception node1) { // if this fails too just continue on to the error display - this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT); - } + // this.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT); + //} + throw new UnsupportedOperationException(); } public void retrieve(String path, File working, Progress progress, String optFilePath) throws Exception { @@ -82,7 +87,6 @@ public void retrieve(String path, File working, Progress progress, String optFil logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + optFilePath); ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + optFilePath); Process p = pb.start(); - // This class is already running in its own thread so it can happily pause until finished. p.waitFor(); @@ -95,6 +99,7 @@ public void retrieve(String path, File working, Progress progress, String optFil throw new Exception("Retrieval of " + working.getName() + " failed. "); } + // FILL IN THE REST OF PROGRESS x dirs, x files, x bytes etc. } @Override diff --git a/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java b/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java index 74b02c33c..ca8ae0fc1 100644 --- a/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java +++ b/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java @@ -162,88 +162,50 @@ public void performAction(Context context) { .withUserId(userID) .withNextState(1)); - // Progress tracking (threaded) - Progress progress = new Progress(); - ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, archiveSize, eventStream); - Thread trackerThread = new Thread(tracker); - trackerThread.start(); + if (archiveFs.hasMultipleCopies()) { + Progress progress = new Progress(); + ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, archiveSize, eventStream); + Thread trackerThread = new Thread(tracker); + trackerThread.start(); + + LOCATION: + for (String location : archiveFs.getLocations()) { + try { + try { + // Ask the driver to copy files to the temp directory + archiveFs.retrieve(archiveId, tarFile, progress, location); + } finally { + // Stop the tracking thread + tracker.stop(); + trackerThread.join(); + } + + this.doRetrieve(depositId, userID, retrievePath, retrieveId, context, userFs, tarFile, eventStream, archiveDigestAlgorithm, archiveDigest, progress); + } catch (Exception e) { + continue LOCATION; + } + break LOCATION; + } + + } else { + // Progress tracking (threaded) + Progress progress = new Progress(); + ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, archiveSize, eventStream); + Thread trackerThread = new Thread(tracker); + trackerThread.start(); - try { - // Ask the driver to copy files to the temp directory - archiveFs.retrieve(archiveId, tarFile, progress); - } finally { - // Stop the tracking thread - tracker.stop(); - trackerThread.join(); - } - - logger.info("Copied: " + progress.dirCount + " directories, " + progress.fileCount + " files, " + progress.byteCount + " bytes"); - - logger.info("Validating data ..."); - eventStream.send(new UpdateProgress(jobID, depositId).withNextState(2) - .withUserId(userID)); - - // Verify integrity with deposit checksum - String systemAlgorithm = Verify.getAlgorithm(); - if (!systemAlgorithm.equals(archiveDigestAlgorithm)) { - throw new Exception("Unsupported checksum algorithm: " + archiveDigestAlgorithm); - } - - String tarHash = Verify.getDigest(tarFile); - logger.info("Checksum algorithm: " + archiveDigestAlgorithm); - logger.info("Checksum: " + tarHash); - - if (!tarHash.equals(archiveDigest)) { - throw new Exception("checksum failed: " + tarHash + " != " + archiveDigest); - } - - // Decompress to the temporary directory - File bagDir = Tar.unTar(tarFile, context.getTempDir()); - long bagDirSize = FileUtils.sizeOfDirectory(bagDir); - - // Validate the bagit directory - if (!Packager.validateBag(bagDir)) { - throw new Exception("Bag is invalid"); + try { + // Ask the driver to copy files to the temp directory + archiveFs.retrieve(archiveId, tarFile, progress); + } finally { + // Stop the tracking thread + tracker.stop(); + trackerThread.join(); + } + + this.doRetrieve(depositId, userID, retrievePath, retrieveId, context, userFs, tarFile, eventStream, archiveDigestAlgorithm, archiveDigest, progress); } - // Get the payload data directory - File payloadDir = bagDir.toPath().resolve("data").toFile(); - long payloadSize = FileUtils.sizeOfDirectory(payloadDir); - - // Copy the extracted files to the target retrieve area - logger.info("Copying to user directory ..."); - eventStream.send(new UpdateProgress(jobID, depositId, 0, bagDirSize, "Starting transfer ...") - .withUserId(userID) - .withNextState(3)); - - // Progress tracking (threaded) - progress = new Progress(); - tracker = new ProgressTracker(progress, jobID, depositId, bagDirSize, eventStream); - trackerThread = new Thread(tracker); - trackerThread.start(); - - try { - ArrayList contents = new ArrayList(Arrays.asList(payloadDir.listFiles())); - for(File content: contents){ - userFs.store(retrievePath, content, progress); - } - } finally { - // Stop the tracking thread - tracker.stop(); - trackerThread.join(); - } - - logger.info("Copied: " + progress.dirCount + " directories, " + progress.fileCount + " files, " + progress.byteCount + " bytes"); - - // Cleanup - logger.info("Cleaning up ..."); - FileUtils.deleteDirectory(bagDir); - tarFile.delete(); - - logger.info("Data retrieve complete: " + retrievePath); - eventStream.send(new RetrieveComplete(jobID, depositId, retrieveId).withNextState(4) - .withUserId(userID)); - } catch (Exception e) { String msg = "Data retrieve failed: " + e.getMessage(); logger.error(msg, e); @@ -251,4 +213,74 @@ public void performAction(Context context) { .withUserId(userID)); } } + + private void doRetrieve(String depositId, String userID, String retrievePath, String retrieveId, Context context, Device userFs, File tarFile, EventSender eventStream, + String archiveDigestAlgorithm, String archiveDigest, Progress progress) throws Exception{ + logger.info("Copied: " + progress.dirCount + " directories, " + progress.fileCount + " files, " + progress.byteCount + " bytes"); + + logger.info("Validating data ..."); + eventStream.send(new UpdateProgress(jobID, depositId).withNextState(2) + .withUserId(userID)); + + // Verify integrity with deposit checksum + String systemAlgorithm = Verify.getAlgorithm(); + if (!systemAlgorithm.equals(archiveDigestAlgorithm)) { + throw new Exception("Unsupported checksum algorithm: " + archiveDigestAlgorithm); + } + + String tarHash = Verify.getDigest(tarFile); + logger.info("Checksum algorithm: " + archiveDigestAlgorithm); + logger.info("Checksum: " + tarHash); + + if (!tarHash.equals(archiveDigest)) { + throw new Exception("checksum failed: " + tarHash + " != " + archiveDigest); + } + + // Decompress to the temporary directory + File bagDir = Tar.unTar(tarFile, context.getTempDir()); + long bagDirSize = FileUtils.sizeOfDirectory(bagDir); + + // Validate the bagit directory + if (!Packager.validateBag(bagDir)) { + throw new Exception("Bag is invalid"); + } + + // Get the payload data directory + File payloadDir = bagDir.toPath().resolve("data").toFile(); + long payloadSize = FileUtils.sizeOfDirectory(payloadDir); + + // Copy the extracted files to the target retrieve area + logger.info("Copying to user directory ..."); + eventStream.send(new UpdateProgress(jobID, depositId, 0, bagDirSize, "Starting transfer ...") + .withUserId(userID) + .withNextState(3)); + + // Progress tracking (threaded) + progress = new Progress(); + ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, bagDirSize, eventStream); + Thread trackerThread = new Thread(tracker); + trackerThread.start(); + + try { + ArrayList contents = new ArrayList(Arrays.asList(payloadDir.listFiles())); + for(File content: contents){ + userFs.store(retrievePath, content, progress); + } + } finally { + // Stop the tracking thread + tracker.stop(); + trackerThread.join(); + } + + logger.info("Copied: " + progress.dirCount + " directories, " + progress.fileCount + " files, " + progress.byteCount + " bytes"); + + // Cleanup + logger.info("Cleaning up ..."); + FileUtils.deleteDirectory(bagDir); + tarFile.delete(); + + logger.info("Data retrieve complete: " + retrievePath); + eventStream.send(new RetrieveComplete(jobID, depositId, retrieveId).withNextState(4) + .withUserId(userID)); + } } From 21d354ab36eb4c4c4e311d5639324c3a42d4ec40 Mon Sep 17 00:00:00 2001 From: David Speed Date: Fri, 16 Feb 2018 16:08:47 +0000 Subject: [PATCH 5/5] IssueId #402 Updated the TSM retrieve command so that if the first TSM copies checksum is wrong we try again using the second copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixed some issues that I discovered during my manual testing, I’ll hopefully manage to add unit tests soon. --- .../storage/impl/TivoliStorageManager.java | 4 +++- .../worker/tasks/Retrieve.java | 21 +++++++++++++--- .../impl/TivoliStorageManagerTest.java | 24 ++++++++++++++++--- 3 files changed, 42 insertions(+), 7 deletions(-) diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java index d54bff388..13b1c6724 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java @@ -82,10 +82,12 @@ public void retrieve(String path, File working, Progress progress) throws Except throw new UnsupportedOperationException(); } + @Override public void retrieve(String path, File working, Progress progress, String optFilePath) throws Exception { logger.info("Retrieve command is " + "dsmc " + " retrieve " + working.getAbsolutePath() + " -optfile=" + optFilePath); - ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + optFilePath); + + ProcessBuilder pb = new ProcessBuilder("dsmc", "retrieve", working.getAbsolutePath(), "-optfile=" + optFilePath, "-replace=true"); Process p = pb.start(); // This class is already running in its own thread so it can happily pause until finished. p.waitFor(); diff --git a/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java b/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java index ca8ae0fc1..92c781038 100644 --- a/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java +++ b/datavault-worker/src/main/java/org/datavaultplatform/worker/tasks/Retrieve.java @@ -5,6 +5,8 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; +import java.util.List; import java.util.Map; import org.apache.commons.io.FileUtils; @@ -163,13 +165,17 @@ public void performAction(Context context) { .withNextState(1)); if (archiveFs.hasMultipleCopies()) { + logger.info("Device has multiple copies"); Progress progress = new Progress(); ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, archiveSize, eventStream); Thread trackerThread = new Thread(tracker); trackerThread.start(); + List locations = archiveFs.getLocations(); + Iterator locationsIt = locations.iterator(); LOCATION: - for (String location : archiveFs.getLocations()) { + while (locationsIt.hasNext()) { + String location = locationsIt.next(); try { try { // Ask the driver to copy files to the temp directory @@ -180,14 +186,22 @@ public void performAction(Context context) { trackerThread.join(); } + logger.info("Attempting retrieve on archive from " + location); this.doRetrieve(depositId, userID, retrievePath, retrieveId, context, userFs, tarFile, eventStream, archiveDigestAlgorithm, archiveDigest, progress); + logger.info("Completed retrieve on archive from " + location); + break LOCATION; } catch (Exception e) { - continue LOCATION; + //if last location has an error throw the error else go round again + //continue LOCATION; + if (!locationsIt.hasNext()) { + logger.info("All locations had problems throwing exception " + e.getMessage()); + throw e; + } } - break LOCATION; } } else { + logger.info("Single copy device"); // Progress tracking (threaded) Progress progress = new Progress(); ProgressTracker tracker = new ProgressTracker(progress, jobID, depositId, archiveSize, eventStream); @@ -233,6 +247,7 @@ private void doRetrieve(String depositId, String userID, String retrievePath, St logger.info("Checksum: " + tarHash); if (!tarHash.equals(archiveDigest)) { + logger.info("Checksum failed: " + tarHash + " != " + archiveDigest); throw new Exception("checksum failed: " + tarHash + " != " + archiveDigest); } diff --git a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java index 8a1426fff..937a9256c 100644 --- a/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java +++ b/datavault-worker/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java @@ -148,7 +148,7 @@ public void testRetrieveValidParams() { // // retrieve from TSM node 1 // System.out.println("Retrieve the tar"); // //dsmc retrieve /tmp/datavault/temp/2848@ubuntu-xenial/513c2b11-30df-4947-846b-a64309c61eb8.tar -// tsm.retrieve(path, working, progress); +// tsm.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT); // // // check that the /tmp/"retval" dir now exists // System.out.println("Check the retrieve archive exits"); @@ -175,7 +175,7 @@ public void testRetrieveValidParams() { } @Test - public void testRetrieveNonExistantFile() { + public void testRetrieveNonExistantFileFromNodeOne() { // retrieve from TSM // Progress progress = new Progress(); // File working = new File("/tmp/test.tar"); @@ -184,7 +184,25 @@ public void testRetrieveNonExistantFile() { // try { // System.out.println("Retrieve the tar"); // //dsmc retrieve /tmp/datavault/temp/2848@ubuntu-xenial/513c2b11-30df-4947-846b-a64309c61eb8.tar -// tsm.retrieve(path, working, progress); +// tsm.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE1_OPT); +// fail("Exception should have been thrown"); +// } catch (Exception e) { +// assertNotNull("Exception should not be null", e); +// assertEquals("Message not as expected", "Retrieval of test.tar failed. ", e.getMessage()); +// } + } + + @Test + public void testRetrieveNonExistantFileFromNodeTwo() { + // retrieve from TSM +// Progress progress = new Progress(); +// File working = new File("/tmp/test.tar"); +// String path = "/tmp"; +// +// try { +// System.out.println("Retrieve the tar"); +// //dsmc retrieve /tmp/datavault/temp/2848@ubuntu-xenial/513c2b11-30df-4947-846b-a64309c61eb8.tar +// tsm.retrieve(path, working, progress, TivoliStorageManager.TSM_SERVER_NODE2_OPT); // fail("Exception should have been thrown"); // } catch (Exception e) { // assertNotNull("Exception should not be null", e);